diff --git a/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs b/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs new file mode 100644 index 000000000..667f3b6b3 --- /dev/null +++ b/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs @@ -0,0 +1,22 @@ +//! Whether an S3 store points at a custom endpoint, which a warehouse reading S3 itself would not follow. + +use crate::store_paths::models::StoreKind; + +/// Why an S3 store points somewhere a warehouse reading S3 itself would not read: an endpoint in the connection or in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows. +pub(in crate::adapters) fn custom_endpoint( + store: &StoreKind, + variable: &dyn Fn(&str) -> Option, +) -> Option { + let from_env = ["AWS_ENDPOINT", "AWS_ENDPOINT_URL"] + .iter() + .any(|name| variable(name).is_some_and(|value| !value.is_empty())); + match store { + StoreKind::S3 { + endpoint: Some(_), .. + } => Some("the S3 connection sets a custom endpoint".to_owned()), + StoreKind::S3 { .. } if from_env => { + Some("AWS_ENDPOINT or AWS_ENDPOINT_URL sets a custom S3 endpoint".to_owned()) + } + _ => None, + } +} diff --git a/crates/sqlload/src/adapters/contract/main/mod.rs b/crates/sqlload/src/adapters/contract/main/mod.rs index 1d80cd6b2..b5fc0a7f7 100644 --- a/crates/sqlload/src/adapters/contract/main/mod.rs +++ b/crates/sqlload/src/adapters/contract/main/mod.rs @@ -1,4 +1,5 @@ pub(in crate::adapters) mod _batch_direct_problem; +pub(in crate::adapters) mod _custom_endpoint; pub(in crate::adapters) mod _dropped_ordinals; pub(in crate::adapters) mod _files_direct_problem; pub(in crate::adapters) mod _footer_rows; diff --git a/crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs b/crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs new file mode 100644 index 000000000..c9696345e --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs @@ -0,0 +1,13 @@ +//! How SQLLoad compares folders with Databricks' external locations. + +/// An external location's `url` as SQLLoad compares folders with it: `s3://` for the `s3a://` and `s3n://` it may be stored as, ending in `/` so that it only covers what lies below it. +pub(crate) fn location_url(url: &str) -> String { + let url = url.trim_end_matches('/'); + let s3 = ["s3a://", "s3n://"] + .iter() + .find_map(|scheme| url.strip_prefix(scheme)); + match s3 { + Some(rest) => format!("s3://{rest}/"), + None => format!("{url}/"), + } +} diff --git a/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs b/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs index f149669a3..b858ed8da 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs @@ -1 +1,2 @@ pub(crate) mod folders; +pub(crate) mod locations; diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs index 5d2d686c2..4f6058082 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs @@ -9,28 +9,18 @@ use crate::adapters::contract::types::SqlAdapter; use crate::adapters::databricks::constants::COPY_OPTIONS; use crate::adapters::databricks::models::DatabricksSql; use crate::column_types::models::ColumnType; +use crate::schema_plan::models::DesiredColumn; -/// `COPY INTO` the batch's table (the staging or build table, or the table itself) from `folder`, each column cast to the type it lands as: the table's own types straight into it or a build table, JSON parsed; text for JSON and UUIDs in a staging table. +/// `COPY INTO` the batch's table (the staging or build table, or the table itself) from `folder`, each column cast to the type it lands as (see [`typed_value`]). pub(crate) fn copy_statement(target: &WriteTarget<'_>, folder: &str) -> String { let adapter = DatabricksSql; let table = target.staging.unwrap_or(target.table); - let final_types = target.direct || target.builds; let selected: Vec = target .columns .iter() .map(|column| { let name = adapter.render_identifier(&column.name); - match (final_types, &column.column_type) { - (true, ColumnType::Json) => format!("parse_json({name}) AS {name}"), - (true, column_type) => format!( - "CAST({name} AS {}) AS {name}", - adapter.render_loader_logical_type(column_type).sql - ), - (false, column_type) => format!( - "CAST({name} AS {}) AS {name}", - adapter.render_staging_type(column_type) - ), - } + format!("{} AS {name}", typed_value(target, column, &name)) }) .collect(); format!( @@ -41,6 +31,22 @@ pub(crate) fn copy_statement(target: &WriteTarget<'_>, folder: &str) -> String { ) } +/// `value` as `column` lands in `target`'s batch table: in the table's own type straight into it or a build table, JSON parsed; in a staging table, text for JSON and UUIDs. +pub(crate) fn typed_value(target: &WriteTarget<'_>, column: &DesiredColumn, value: &str) -> String { + let adapter = DatabricksSql; + match (target.direct || target.builds, &column.column_type) { + (true, ColumnType::Json) => format!("parse_json({value})"), + (true, column_type) => format!( + "CAST({value} AS {})", + adapter.render_loader_logical_type(column_type).sql + ), + (false, column_type) => format!( + "CAST({value} AS {})", + adapter.render_staging_type(column_type) + ), + } +} + /// `batch` with its time-of-day columns as text, since Databricks reads no Parquet `TIME` and stores times as text. pub(crate) fn readable_batch(batch: RecordBatch) -> Result { let schema = batch.schema(); diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs new file mode 100644 index 000000000..805a3e705 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs @@ -0,0 +1,227 @@ +//! A files batch loaded by Databricks `COPY`ing the source's own Parquet files from S3. + +use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem; +use crate::adapters::contract::main::_footer_rows::footer_rows; +use crate::adapters::contract::models::{DirectBatch, DirectLoaded, TableName}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::s3_copy::{ + S3Batch, S3File, copy_groups, s3_copy_statement, +}; +use crate::adapters::databricks::constants::{INSERTED_ROWS_COLUMN, MAX_FILES_PER_COPY}; +use crate::adapters::databricks::main::stage::_direct_problem::direct_problem; +use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter}; +use crate::databricks_api::main::execute_statement::execute_statement; +use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN}; +use crate::files_source::main::normal_value::normal_value; +use crate::files_source::models::DataFile; +use crate::load_naming::main::metadata_name::metadata_name; +use crate::load_naming::models::MetadataRole; +use crate::loader::errors::LoaderError; +use crate::schema_plan::models::DesiredColumn; +use crate::sql_api::models::SqlRows; +use arrow_schema::Schema; + +impl DatabricksWriter { + /// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, into the staging or build table the load publishes from, or into the table itself: one `COPY INTO` (one transaction) per [`copy_groups`] group, as `FILES` takes at most 1000 files and `mergeSchema` won't merge one column held as two types. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose inserted rows, as `COPY INTO` reports them, differ from its footers'; an append into the table itself first takes back what it committed. Files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location. + pub(in crate::adapters) async fn copy_files( + &mut self, + batch: &DirectBatch<'_>, + ) -> Result, LoaderError> { + let Some(source) = batch.spec.files.as_ref() else { + return Ok(None); + }; + let folder = source.location.to_string(); + let covered = self + .destination + .external_locations + .get() + .is_some_and(|locations| locations.iter().any(|url| folder.starts_with(url.as_str()))); + if !covered { + return Ok(None); + } + if let Some(problem) = + batch_direct_problem(batch, source, direct_problem(batch.spec, batch.read)) + { + return Err(LoaderError::Invalid(format!( + "Databricks cannot load this batch's files exactly by reading them itself, and they never go through SQLLoad instead: {problem}" + ))); + } + let target = &batch.target; + let (into, columns) = match (target.staging, target.builds) { + (Some(staging), false) => (staging, target.columns), + (staging, _) => (staging.unwrap_or(target.table), batch.write_columns), + }; + let no_footer = Schema::empty(); + let schemas: Vec<(&DataFile, &Schema)> = batch + .files + .iter() + .map(|data| { + let schema = batch + .footers + .parquet + .get(&data.file.store_key()) + .map_or(&no_footer, |footer| footer.schema().as_ref()); + (data, schema) + }) + .collect(); + let mut inserted = 0; + let mut copied = false; + for group in copy_groups(schemas, MAX_FILES_PER_COPY) { + let statement = group_statement(batch, &group, (into, columns), folder.clone())?; + let result = match execute_statement(&self.destination.api, &statement).await { + Ok(result) => { + copied = true; + inserted_rows(&result) + } + Err(error) => Err(error.into()), + }; + match result { + Ok(rows) => inserted += rows, + Err(error) if copied => return Err(self.undo(batch, error).await), + Err(error) => return Err(error), + } + } + let expected = footer_rows(batch.footers); + if inserted != expected { + let error = LoaderError::Invalid(format!( + "Databricks loaded {inserted} rows from this batch's files, but their footers hold {expected}" + )); + return Err(self.undo(batch, error).await); + } + Ok(Some(match target.staging { + Some(_) => DirectLoaded::Staged(expected), + None => DirectLoaded::Committed(expected), + })) + } +} + +impl DatabricksWriter { + /// After `error`, take back the rows a batch going straight into the table committed: its load's rows from its files, which a later run would otherwise skip as loaded. A batch landing in a staging table needs nothing, as the load drops it. + async fn undo(&self, batch: &DirectBatch<'_>, error: LoaderError) -> LoaderError { + let target = &batch.target; + if target.staging.is_some() { + return error; + } + let Some(source) = batch.spec.files.as_ref() else { + return error; + }; + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let paths: Vec = batch + .files + .iter() + .map(|data| literal(&format!("{}{}", source.location, data.file.relative_key))) + .collect(); + let statement = format!( + "DELETE FROM {} WHERE {} = {} AND {} IN ({})", + DatabricksSql.render_qualified_name(target.table), + DatabricksSql.render_identifier(metadata_name(target.compat, MetadataRole::LoadId)), + literal(target.load_id), + DatabricksSql.render_identifier(SOURCE_FILE_COLUMN), + paths.join(", ") + ); + match execute_statement(&self.destination.api, &statement).await { + Ok(_) => error, + Err(undo) => LoaderError::Invalid(format!( + "{error}; taking back the rows it committed then failed too, so the table holds them: {undo}" + )), + } + } +} + +/// The `COPY INTO` that loads `group`, files of `batch` below `folder`, into `into`'s `columns`, reading each table column the group's files hold by its name in them. +fn group_statement( + batch: &DirectBatch<'_>, + group: &[&DataFile], + (into, columns): (&TableName, &[DesiredColumn]), + folder: String, +) -> Result { + let mut files = Vec::with_capacity(group.len()); + let mut in_files: Vec<(String, String)> = Vec::new(); + for data in group { + files.push(s3_file(batch, data, columns)?); + let Some(footer) = batch.footers.parquet.get(&data.file.store_key()) else { + continue; + }; + for field in footer.schema().fields() { + let loaded = batch.spec.source_table.loaded_as(field.name()).to_owned(); + if !in_files.iter().any(|(name, _)| *name == loaded) { + in_files.push((loaded, field.name().clone())); + } + } + } + Ok(s3_copy_statement( + (&DatabricksSql.render_qualified_name(into), columns), + &batch.target, + &S3Batch { + folder, + files, + in_files, + }, + batch.loaded_at, + )) +} + +/// The rows a `COPY INTO`'s `result` reports it inserted. +fn inserted_rows(result: &SqlRows) -> Result { + let reported = result + .columns + .iter() + .position(|column| column == INSERTED_ROWS_COLUMN) + .and_then(|index| result.rows.first()?.get(index)?.clone()); + let Some(text) = reported else { + return Err(LoaderError::Invalid(format!( + "COPY INTO reported no {INSERTED_ROWS_COLUMN}" + ))); + }; + text.parse::().map_err(|error| { + LoaderError::Invalid(format!( + "COPY INTO reported {INSERTED_ROWS_COLUMN} = {text:?}: {error}" + )) + }) +} + +/// The values SQLLoad gives every row of `data`: its name and version as the files source records them, and its partition values (constants among them), each converted to its column's type as SQLLoad's reader would, so a value that does not convert fails here rather than in Databricks' cast. +fn s3_file( + batch: &DirectBatch<'_>, + data: &DataFile, + columns: &[DesiredColumn], +) -> Result { + let Some(source) = batch.spec.files.as_ref() else { + return Err(LoaderError::Invalid("not a files table".to_owned())); + }; + let mut constants = vec![ + ( + SOURCE_FILE_COLUMN.to_owned(), + Some(format!("{}{}", source.location, data.file.relative_key)), + ), + (SOURCE_VERSION_COLUMN.to_owned(), Some(data.file.version())), + ]; + for (name, _) in &source.partition_types { + let value = data + .partitions + .iter() + .find(|(partition, _)| partition == name) + .map(|(_, value)| value.clone()); + let loaded = batch.spec.source_table.loaded_as(name); + let column_type = columns + .iter() + .find(|column| column.name == loaded) + .map(|column| &column.column_type); + let value = match (value, column_type) { + (Some(value), Some(column_type)) => { + Some(normal_value(&value, column_type).map_err(|reason| { + LoaderError::Invalid(format!( + "{}: {name} = {value:?} does not load as {column_type}: {reason}", + data.file.relative_key + )) + })?) + } + (value, _) => value, + }; + constants.push((loaded.to_owned(), value)); + } + Ok(S3File { + key: data.file.relative_key.clone(), + constants, + }) +} diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs index 256e51289..30bac9520 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod copy_select; +pub(crate) mod direct_load; +pub(crate) mod s3_copy; pub(crate) mod table_writer; diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs new file mode 100644 index 000000000..ea13bf884 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs @@ -0,0 +1,143 @@ +//! The `COPY INTO` that has Databricks read a files batch's Parquet files where they lie in S3. + +use crate::adapters::contract::models::WriteTarget; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::copy_select::typed_value; +use crate::adapters::databricks::constants::S3_COPY_OPTIONS; +use crate::adapters::databricks::models::DatabricksSql; +use crate::files_source::constants::SOURCE_ROW_COLUMN; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::schema_plan::models::DesiredColumn; +use arrow_schema::{Field, Schema}; + +/// The path `COPY INTO` reads each row from, as SQLLoad names it: `_metadata.file_path` is percent-encoded, except for `+`, which `url_decode` alone would read as a space. +const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))"; + +/// One batch read from S3: the folder `COPY INTO` reads (`s3://bucket/prefix/`), and for each file its key below it and the values SQLLoad gives every row of it (`None` for NULL), by column. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct S3Batch { + pub folder: String, + pub files: Vec, + /// Each table column the batch's files hold, with its name in them, which `naming`, `rename` or numbering may have changed. + pub in_files: Vec<(String, String)>, +} + +/// One file of an [`S3Batch`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct S3File { + pub key: String, + pub constants: Vec<(String, Option)>, +} + +/// `COPY INTO into` every one of `columns`, typed as `target` lands them, from `batch`'s files: a column takes the file's own value when SQLLoad gives it one (picked by the file's path, and a row from any other path fails the statement), the files' column when they hold it, and NULL otherwise; `_sqlload_source_row` numbers each file's rows from 1 in stored order; the load id and load time are the batch's. +pub(crate) fn s3_copy_statement( + (into, columns): (&str, &[DesiredColumn]), + target: &WriteTarget<'_>, + batch: &S3Batch, + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let mut values: Vec = Vec::with_capacity(columns.len()); + for column in columns { + values.push(format!( + "{} AS {}", + typed_value( + target, + column, + &column_value(target, batch, column, loaded_at) + ), + DatabricksSql.render_identifier(&column.name) + )); + } + let keys: Vec = batch.files.iter().map(|file| literal(&file.key)).collect(); + format!( + "COPY INTO {into} FROM (SELECT {} FROM {}) FILEFORMAT = PARQUET FILES = ({}) {S3_COPY_OPTIONS}", + values.join(", "), + literal(&batch.folder), + keys.join(", ") + ) +} + +/// Where `column`'s value comes from in `batch`, before it is typed: the value SQLLoad gives each file, the load id or load time, the row's position in its file, the files' own column, or NULL when no file holds it. +fn column_value( + target: &WriteTarget<'_>, + batch: &S3Batch, + column: &DesiredColumn, + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let mut given: Vec<(&S3File, Option<&String>)> = Vec::new(); + for file in &batch.files { + if let Some((_, value)) = file.constants.iter().find(|(name, _)| *name == column.name) { + given.push((file, value.as_ref())); + } + } + let held = batch.in_files.iter().find(|(name, _)| *name == column.name); + match (metadata_role(target.compat, &column.name), held) { + _ if !given.is_empty() => per_file(batch, &given), + (Some(MetadataRole::LoadId), _) => literal(target.load_id), + (Some(MetadataRole::LoadedAt), _) => literal(loaded_at), + _ if column.name == SOURCE_ROW_COLUMN => "_metadata.row_index + 1".to_owned(), + (_, Some((_, held))) => DatabricksSql.render_exact_identifier(held), + (_, None) => "NULL".to_owned(), + } +} + +/// The value each of `given`'s files gives a column, picked by the path a row was read from; a path not in the batch fails the statement rather than load a row without it. +fn per_file(batch: &S3Batch, given: &[(&S3File, Option<&String>)]) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let picks: Vec = given + .iter() + .map(|(file, value)| { + format!( + "WHEN {} THEN {}", + literal(&format!("{}{}", batch.folder, file.key)), + value.map_or_else(|| "NULL".to_owned(), |value| literal(value)) + ) + }) + .collect(); + format!( + "CASE {FILE_PATH} {} ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END", + picks.join(" ") + ) +} + +/// `files`, in order, in the `COPY INTO`s that load them: at most `limit` to each (the most `FILES` takes), and never two that hold one column as different types, which `mergeSchema` refuses to merge (an INT32 in one file and an INT64 in another). +pub(crate) fn copy_groups(files: Vec<(T, &Schema)>, limit: usize) -> Vec> { + let mut groups: Vec<(Vec, Vec<&Field>)> = Vec::new(); + for (file, schema) in files { + let joins = groups + .iter() + .position(|(members, held)| members.len() < limit && agrees(held, schema)); + let index = joins.unwrap_or_else(|| { + groups.push((Vec::new(), Vec::new())); + groups.len() - 1 + }); + let (members, held) = &mut groups[index]; + members.push(file); + for field in schema.fields() { + if !held + .iter() + .any(|other| other.name().eq_ignore_ascii_case(field.name())) + { + held.push(field); + } + } + } + groups.into_iter().map(|(members, _)| members).collect() +} + +/// Whether `schema` holds every column it shares with `held` as the same type, matching names in any case as Databricks does. +fn agrees(held: &[&Field], schema: &Schema) -> bool { + for field in schema.fields() { + for other in held { + if other.name().eq_ignore_ascii_case(field.name()) + && other.data_type() != field.data_type() + { + return false; + } + } + } + true +} diff --git a/crates/sqlload/src/adapters/databricks/constants.rs b/crates/sqlload/src/adapters/databricks/constants.rs index b6f1565a9..62579e920 100644 --- a/crates/sqlload/src/adapters/databricks/constants.rs +++ b/crates/sqlload/src/adapters/databricks/constants.rs @@ -22,5 +22,24 @@ pub const TRANSACTION_STATEMENTS: &[&str] = &["BEGIN", "COMMIT", "ROLLBACK"]; /// How staged Parquet files are read: every file again, even one a `COPY INTO` has loaded before. pub const COPY_OPTIONS: &str = "FILEFORMAT = PARQUET COPY_OPTIONS ('force' = 'true')"; +/// How a batch's files are read where they lie in S3, after its `FILES` list: every column any of them holds (one lacking it gives NULL), every file again, even one a `COPY INTO` has loaded before. +pub const S3_COPY_OPTIONS: &str = + "FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')"; + +/// The most files one `COPY INTO`'s `FILES` list takes. +pub const MAX_FILES_PER_COPY: usize = 1000; + +/// The external locations the service principal can see. +pub const LIST_LOCATIONS: &str = "SHOW EXTERNAL LOCATIONS"; + +/// The column of [`LIST_LOCATIONS`] holding each location's URL. +pub const LOCATION_URL_COLUMN: &str = "url"; + +/// The column of a `COPY INTO`'s result holding the rows it inserted. +pub const INSERTED_ROWS_COLUMN: &str = "num_inserted_rows"; + +/// Why `plan` refuses Parquet files in S3 that no external location covers, after their folder. +pub const OUTSIDE_LOCATIONS: &str = "cannot be read by Databricks: no Unity Catalog external location covers it, and Parquet files in S3 never go through SQLLoad into Databricks. Fix: create an external location whose URL holds the folder (CREATE EXTERNAL LOCATION ... URL 's3://bucket/folder/' WITH (STORAGE CREDENTIAL ...)), and grant the service principal READ FILES on it"; + /// Shown for `copy_to_internal_stage` on a Databricks target. pub const NO_INTERNAL_STAGE_COPY: &str = "copy_to_internal_stage is for Snowflake targets; Databricks loads the files through its staging volume"; diff --git a/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs b/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs index e01755732..ee7048b0f 100644 --- a/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs +++ b/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs @@ -41,5 +41,6 @@ pub(in crate::adapters) fn connect_databricks( catalog: catalog.unwrap_or(own_catalog).to_owned(), volume: volume_path(volume), ddl: Arc::default(), + external_locations: Arc::default(), }) } diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_check_folder.rs b/crates/sqlload/src/adapters/databricks/main/stage/_check_folder.rs new file mode 100644 index 000000000..08a25b38f --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/main/stage/_check_folder.rs @@ -0,0 +1,20 @@ +//! Check that Databricks can read a folder of files through its external location. + +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::models::{DatabricksDestination, DatabricksSql}; +use crate::databricks_api::main::execute_statement::execute_statement; +use crate::http::errors::ApiError; + +/// Check that `destination`'s service principal can read `folder`, which a location it sees covers: seeing a location does not grant READ FILES on it, which `LIST` needs as `COPY INTO` does. +pub(in crate::adapters) async fn check_folder( + destination: &DatabricksDestination, + folder: &str, +) -> Result<(), ApiError> { + let statement = format!( + "LIST {} LIMIT 1", + DatabricksSql.render_loader_value_literal(folder) + ); + execute_statement(&destination.api, &statement) + .await + .map(|_| ()) +} diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs b/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs new file mode 100644 index 000000000..a27ba6f07 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs @@ -0,0 +1,46 @@ +//! Whether Databricks may load a files table's files itself. + +use crate::adapters::contract::main::_custom_endpoint::custom_endpoint; +use crate::adapters::contract::main::_files_direct_problem::files_direct_problem; +use crate::column_types::models::ColumnType; +use crate::files_source::models::{FilesRead, SourceFormat}; +use crate::loader::models::TableSpec; +use crate::store_paths::models::StoreKind; + +/// Why Databricks cannot load the planned `read` of a files table in S3 exactly by `COPY`ing its Parquet files where they lie, so SQLLoad reads them into the staging volume instead; `None` when it can, or when the files aren't in S3. Every column must load as the type the files hold it as, so Databricks' casts never differ from SQLLoad's; an external location reaches only AWS S3, not an S3-compatible endpoint; masks need SQLLoad's reader; and Databricks reads a Parquet time as text in its own format, a UUID as bytes, and lists as its own arrays. +pub(in crate::adapters) fn direct_problem(spec: &TableSpec, read: &FilesRead) -> Option { + let files = spec.files.as_ref()?; + let StoreKind::S3 { .. } = &files.location.store else { + return None; + }; + let problems = [ + (files.encoding.format != SourceFormat::Parquet) + .then(|| "only Parquet files are read by Databricks itself".to_owned()), + custom_endpoint(&files.location.store, &environment_variable).map(|endpoint| { + format!("{endpoint}, which no Unity Catalog external location reaches") + }), + (!spec.masked.columns.is_empty()).then(|| "the table masks columns".to_owned()), + files_direct_problem(files, read), + read.columns.iter().find_map(|column| { + let held = match &column.column_type { + ColumnType::Time => { + "a time, which Databricks reads from Parquet as text in its own format" + } + ColumnType::Uuid => "a UUID, which Databricks reads from Parquet as bytes", + ColumnType::Array(_) => "a list, which Databricks reads as its own array", + ColumnType::Native(_) => "a type only SQLLoad's reader converts", + _ => return None, + }; + Some(format!("{} is {held}", column.name)) + }), + ]; + problems.into_iter().flatten().next() +} + +/// `name`'s value in the process environment; unset when it isn't Unicode, as SQLLoad's reader takes it. +fn environment_variable(name: &str) -> Option { + match std::env::var(name) { + Ok(value) => Some(value), + Err(std::env::VarError::NotPresent | std::env::VarError::NotUnicode(_)) => None, + } +} diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs new file mode 100644 index 000000000..f39a66d1e --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs @@ -0,0 +1,34 @@ +//! The Unity Catalog external locations a Databricks destination reads files through. + +use crate::adapters::databricks::_helpers::stage::locations::location_url; +use crate::adapters::databricks::constants::{LIST_LOCATIONS, LOCATION_URL_COLUMN}; +use crate::adapters::databricks::models::DatabricksDestination; +use crate::databricks_api::main::execute_statement::execute_statement; +use crate::http::errors::ApiError; + +/// The URLs of the external locations `destination`'s service principal can see, each ending in `/` so that a folder only covers what lies below it; listed once per run. +pub(in crate::adapters) async fn external_locations( + destination: &DatabricksDestination, +) -> Result<&[String], ApiError> { + let listed = destination + .external_locations + .get_or_try_init(|| async { + let rows = execute_statement(&destination.api, LIST_LOCATIONS).await?; + let url = rows + .columns + .iter() + .position(|column| column == LOCATION_URL_COLUMN) + .ok_or_else(|| { + ApiError::Protocol(format!("{LIST_LOCATIONS} returned no url column")) + })?; + Ok::, ApiError>( + rows.rows + .iter() + .filter_map(|row| row.get(url).cloned().flatten()) + .map(|url| location_url(&url)) + .collect(), + ) + }) + .await?; + Ok(listed) +} diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs b/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs index dc3dca5fa..43c8e0317 100644 --- a/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs +++ b/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs @@ -1,16 +1,27 @@ //! What Databricks takes. +use crate::adapters::databricks::constants::OUTSIDE_LOCATIONS; use crate::adapters::databricks::models::DatabricksDestination; use crate::routing::models::{FileFormat, Store, WriterAccepts}; -/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads. +/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads; and Parquet files in S3 under one of its external locations, which `COPY INTO` reads where they lie. Parquet files in S3 no external location covers are refused, never read by SQLLoad. pub(in crate::adapters) fn writer_accepts(destination: &DatabricksDestination) -> WriterAccepts { let volume = Store::InternalStage(destination.volume.clone()); + let locations = destination + .external_locations + .get() + .into_iter() + .flatten() + .map(|url| Store::ObjectStore(url.clone())); WriterAccepts { arrow: false, - pulls_from: vec![volume.clone()], + pulls_from: std::iter::once(volume.clone()).chain(locations).collect(), pull_formats: vec![FileFormat::Parquet], staging: vec![volume], + must_pull: Some(( + Store::ObjectStore("s3://".to_owned()), + OUTSIDE_LOCATIONS.to_owned(), + )), ..WriterAccepts::default() } } diff --git a/crates/sqlload/src/adapters/databricks/main/stage/mod.rs b/crates/sqlload/src/adapters/databricks/main/stage/mod.rs index 548796b84..55a6d787d 100644 --- a/crates/sqlload/src/adapters/databricks/main/stage/mod.rs +++ b/crates/sqlload/src/adapters/databricks/main/stage/mod.rs @@ -1,4 +1,7 @@ +pub(in crate::adapters) mod _check_folder; pub(in crate::adapters) mod _check_volume; +pub(in crate::adapters) mod _direct_problem; +pub(in crate::adapters) mod _external_locations; pub(in crate::adapters) mod _remove_folder; pub(in crate::adapters) mod _remove_stale; pub(in crate::adapters) mod _writer_accepts; diff --git a/crates/sqlload/src/adapters/databricks/models.rs b/crates/sqlload/src/adapters/databricks/models.rs index 0d4761af2..73e10675f 100644 --- a/crates/sqlload/src/adapters/databricks/models.rs +++ b/crates/sqlload/src/adapters/databricks/models.rs @@ -17,6 +17,8 @@ pub struct DatabricksDestination { pub volume: String, /// Held while a load's tables change the schema, one at a time. pub ddl: Arc>, + /// The URLs (each ending in `/`) of the Unity Catalog external locations it reads files through, listed once, at plan, when a table's files are in S3. + pub external_locations: Arc>>, } /// One table's write session on Databricks: Parquet parts uploaded to the volume, then loaded by `COPY INTO`. diff --git a/crates/sqlload/src/adapters/databricks/tests.rs b/crates/sqlload/src/adapters/databricks/tests.rs index fb552a6b6..57161de5d 100644 --- a/crates/sqlload/src/adapters/databricks/tests.rs +++ b/crates/sqlload/src/adapters/databricks/tests.rs @@ -2,6 +2,8 @@ mod dialect; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_s3_copy.rs"] +mod s3_copy; #[path = "tests/test_stale_removals.rs"] mod stale_removals; #[path = "tests/test_types.rs"] diff --git a/crates/sqlload/src/adapters/databricks/tests/helpers.rs b/crates/sqlload/src/adapters/databricks/tests/helpers.rs index b37037e68..23c220eb1 100644 --- a/crates/sqlload/src/adapters/databricks/tests/helpers.rs +++ b/crates/sqlload/src/adapters/databricks/tests/helpers.rs @@ -54,3 +54,77 @@ pub(crate) fn listed_load( folder, } } + +/// The `COPY INTO` of two Parquet files under `s3://lake/orders/` (one in the partition folder `day=1/`, holding `2026-11-17`; one, whose name holds a quote and a `+`, with no partition value), batch 2 of load L1: into `main.raw.orders` itself, or into `main.raw.orders_stage` when `staged`. The files hold `id` as `ID`; straight into the table a column `gone` no file holds is written, into the staging table a JSON column `payload` the files hold as `Payload`. +pub(crate) fn s3_copy_sql(staged: bool) -> String { + use crate::adapters::contract::models::WriteTarget; + use crate::adapters::contract::types::SqlAdapter; + use crate::adapters::databricks::_helpers::write::s3_copy::{ + S3Batch, S3File, s3_copy_statement, + }; + use crate::adapters::databricks::models::DatabricksSql; + use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_ROW_COLUMN}; + use crate::load_naming::models::Compat; + + let column = |name: &str, column_type: ColumnType| DesiredColumn { + name: name.to_owned(), + column_type, + origin: ColumnOrigin::Source, + }; + let last = [ + column("gone", ColumnType::Text), + column("payload", ColumnType::Json), + ]; + let columns = vec![ + column("id", ColumnType::Int), + column("day", ColumnType::Date), + column(SOURCE_FILE_COLUMN, ColumnType::Text), + column(SOURCE_ROW_COLUMN, ColumnType::Int), + last[usize::from(staged)].clone(), + column("_sqlload_load_id", ColumnType::Text), + column("_sqlload_loaded_at", ColumnType::TimestampTz), + ]; + let staging = TableName { + table: "orders_stage".to_owned(), + ..table() + }; + let target = WriteTarget { + table: &table(), + staging: Some(&staging).filter(|_| staged), + columns: &columns, + write_columns: &columns, + compat: Compat::Sqlload, + load_id: "L1", + batch: 2, + direct: !staged, + builds: false, + constants: &[], + }; + let file = |key: &str, day: Option<&str>| S3File { + key: key.to_owned(), + constants: vec![ + ("day".to_owned(), day.map(str::to_owned)), + ( + SOURCE_FILE_COLUMN.to_owned(), + Some(format!("s3://lake/orders/{key}")), + ), + ], + }; + let into = DatabricksSql.render_qualified_name(target.staging.unwrap_or(target.table)); + s3_copy_statement( + (&into, &columns), + &target, + &S3Batch { + folder: "s3://lake/orders/".to_owned(), + files: vec![ + file("day=1/a b.parquet", Some("2026-11-17")), + file("o'brien+x.parquet", None), + ], + in_files: vec![ + ("id".to_owned(), "ID".to_owned()), + ("payload".to_owned(), "Payload".to_owned()), + ], + }, + "2026-11-18T00:00:00Z", + ) +} diff --git a/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs new file mode 100644 index 000000000..ce94148de --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs @@ -0,0 +1,137 @@ +use crate::adapters::databricks::_helpers::stage::locations::location_url; +use crate::adapters::databricks::_helpers::write::s3_copy::copy_groups; +use crate::adapters::databricks::tests::helpers::s3_copy_sql; +use crate::adapters::databricks::tests::test_types::{ + CopyGroupsTestCase, LocationUrlTestCase, RenderedTestCase, +}; +use arrow_schema::{DataType, Field, Schema}; + +#[test] +fn given_a_batch_of_parquet_files_in_s3_when_databricks_copies_them_then_each_row_gets_its_files_values_and_the_files_own_columns() + { + let test_cases = [ + RenderedTestCase { + description: "straight into the table: final types, the renamed column read by its name in the files, a column no file holds as NULL, per-file values picked by the decoded path", + rendered: s3_copy_sql(false), + expected_statement: "COPY INTO `main`.`raw`.`orders` FROM (SELECT CAST(`ID` AS BIGINT) AS `id`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN '2026-11-17' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN NULL ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS DATE) AS `day`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN 's3://lake/orders/day=1/a b.parquet' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN 's3://lake/orders/o\\'brien+x.parquet' ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS STRING) AS `_sqlload_source_file`, CAST(_metadata.row_index + 1 AS BIGINT) AS `_sqlload_source_row`, CAST(NULL AS STRING) AS `gone`, CAST('L1' AS STRING) AS `_sqlload_load_id`, CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` FROM 's3://lake/orders/') FILEFORMAT = PARQUET FILES = ('day=1/a b.parquet', 'o\\'brien+x.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + RenderedTestCase { + description: "into the staging table: JSON stays text there, parsed only at publish", + rendered: s3_copy_sql(true), + expected_statement: "COPY INTO `main`.`raw`.`orders_stage` FROM (SELECT CAST(`ID` AS BIGINT) AS `id`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN '2026-11-17' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN NULL ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS DATE) AS `day`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN 's3://lake/orders/day=1/a b.parquet' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN 's3://lake/orders/o\\'brien+x.parquet' ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS STRING) AS `_sqlload_source_file`, CAST(_metadata.row_index + 1 AS BIGINT) AS `_sqlload_source_row`, CAST(`Payload` AS STRING) AS `payload`, CAST('L1' AS STRING) AS `_sqlload_load_id`, CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` FROM 's3://lake/orders/') FILEFORMAT = PARQUET FILES = ('day=1/a b.parquet', 'o\\'brien+x.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + ]; + for test_case in &test_cases { + assert_eq!( + test_case.rendered, test_case.expected_statement, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_batchs_parquet_files_when_databricks_copies_them_then_each_copy_holds_files_whose_shared_columns_agree_and_no_more_than_files_takes() + { + let test_cases = [ + CopyGroupsTestCase { + description: "files that agree, or hold different columns, share one copy", + files: vec![ + ("a", vec![("id", DataType::Int64)]), + ("b", vec![("id", DataType::Int64), ("name", DataType::Utf8)]), + ("c", vec![("name", DataType::Utf8)]), + ], + limit: 1000, + expected_groups: vec![vec!["a", "b", "c"]], + }, + CopyGroupsTestCase { + description: "an INT32 and an INT64 of one column go to separate copies, and a later file joins the first copy it agrees with", + files: vec![ + ("a", vec![("id", DataType::Int32)]), + ("b", vec![("id", DataType::Int64)]), + ("c", vec![("id", DataType::Int32), ("x", DataType::Utf8)]), + ("d", vec![("x", DataType::Int64)]), + ], + limit: 1000, + expected_groups: vec![vec!["a", "c"], vec!["b", "d"]], + }, + CopyGroupsTestCase { + description: "one column named in different cases is one column to Databricks, so different types still split", + files: vec![ + ("a", vec![("ID", DataType::Int32)]), + ("b", vec![("id", DataType::Int64)]), + ], + limit: 1000, + expected_groups: vec![vec!["a"], vec!["b"]], + }, + CopyGroupsTestCase { + description: "no copy holds more files than the limit", + files: vec![ + ("a", vec![("id", DataType::Int64)]), + ("b", vec![("id", DataType::Int64)]), + ("c", vec![("id", DataType::Int64)]), + ], + limit: 2, + expected_groups: vec![vec!["a", "b"], vec!["c"]], + }, + ]; + for test_case in test_cases { + let schemas: Vec<(&str, Schema)> = test_case + .files + .iter() + .map(|(name, columns)| { + let fields: Vec = columns + .iter() + .map(|(column, data_type)| Field::new(*column, data_type.clone(), true)) + .collect(); + (*name, Schema::new(fields)) + }) + .collect(); + let groups = copy_groups( + schemas + .iter() + .map(|(name, schema)| (*name, schema)) + .collect(), + test_case.limit, + ); + assert_eq!( + groups, test_case.expected_groups, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_an_external_locations_url_when_comparing_folders_with_it_then_it_is_an_s3_folder() { + let test_cases = [ + LocationUrlTestCase { + description: "an s3:// URL without a trailing slash", + url: "s3://lake/raw", + expected_url: "s3://lake/raw/", + }, + LocationUrlTestCase { + description: "an s3a:// URL", + url: "s3a://lake/raw/", + expected_url: "s3://lake/raw/", + }, + LocationUrlTestCase { + description: "an s3n:// URL of a whole bucket", + url: "s3n://lake", + expected_url: "s3://lake/", + }, + LocationUrlTestCase { + description: "an Azure URL stays as it is", + url: "abfss://box@account.dfs.core.windows.net/raw", + expected_url: "abfss://box@account.dfs.core.windows.net/raw/", + }, + ]; + for test_case in test_cases { + assert_eq!( + location_url(test_case.url), + test_case.expected_url, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/databricks/tests/test_types.rs b/crates/sqlload/src/adapters/databricks/tests/test_types.rs index 036442524..146d4e041 100644 --- a/crates/sqlload/src/adapters/databricks/tests/test_types.rs +++ b/crates/sqlload/src/adapters/databricks/tests/test_types.rs @@ -20,3 +20,17 @@ pub(crate) struct RenderedTestCase { pub(crate) rendered: String, pub(crate) expected_statement: &'static str, } + +pub(crate) struct CopyGroupsTestCase { + pub(crate) description: &'static str, + /// Each file's name and its columns, by name and type. + pub(crate) files: Vec<(&'static str, Vec<(&'static str, arrow_schema::DataType)>)>, + pub(crate) limit: usize, + pub(crate) expected_groups: Vec>, +} + +pub(crate) struct LocationUrlTestCase { + pub(crate) description: &'static str, + pub(crate) url: &'static str, + pub(crate) expected_url: &'static str, +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs index 90298b3f5..f2bc3fc70 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs @@ -4,6 +4,7 @@ use crate::adapters::bigquery::constants::NO_INTERNAL_STAGE_COPY as BIGQUERY_NO_ use crate::adapters::clickhouse::constants::NO_INTERNAL_STAGE_COPY as CLICKHOUSE_NO_INTERNAL_STAGE_COPY; use crate::adapters::clickhouse::main::plan::_direct_problem::direct_problem as clickhouse_direct_problem; use crate::adapters::databricks::constants::NO_INTERNAL_STAGE_COPY; +use crate::adapters::databricks::main::stage::_direct_problem::direct_problem as databricks_direct_problem; #[cfg(feature = "duckdb")] use crate::adapters::duckdb::main::_direct_problem::direct_problem; use crate::adapters::postgres::constants::NO_INTERNAL_STAGE_COPY as POSTGRES_NO_INTERNAL_STAGE_COPY; @@ -14,7 +15,7 @@ use crate::column_types::models::Warehouse; use crate::files_source::models::FilesRead; use crate::loader::models::TableSpec; -/// Why the destination cannot load the planned `read` of a files table exactly from its files: for DuckDB, by reading them itself wherever they are; for ClickHouse, by reading them from S3; for Snowflake, under the external `stage` that covers them, or copied unchanged to an internal stage (`copy_to_internal_stage`); `None` when it can, or when neither applies. +/// Why the destination cannot load the planned `read` of a files table exactly from its files: for DuckDB, by reading them itself wherever they are; for ClickHouse, by reading them from S3; for Databricks, by `COPY`ing them from S3 through an external location; for Snowflake, under the external `stage` that covers them, or copied unchanged to an internal stage (`copy_to_internal_stage`); `None` when it can, or when neither applies. pub fn direct_blocked( spec: &TableSpec, read: &FilesRead, @@ -32,6 +33,7 @@ pub fn direct_blocked( return match (files.copy_to_internal_stage, spec.warehouse) { (true, _) => Some(refused.to_owned()), (false, Warehouse::ClickHouse) => clickhouse_direct_problem(spec, read), + (false, Warehouse::Databricks) => databricks_direct_problem(spec, read), (false, _) => None, }; } diff --git a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs new file mode 100644 index 000000000..2b741f084 --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs @@ -0,0 +1,32 @@ +//! Where a destination reads files itself, beyond its own stores. + +use crate::adapters::databricks::main::stage::_check_folder::check_folder; +use crate::adapters::databricks::main::stage::_external_locations::external_locations; +use crate::adapters::dispatch::models::Destination; + +/// Look up, once per run, where `destination` reads files itself beyond its own stores, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers: Databricks' Unity Catalog external locations; and check that it can read `folder`, which holds files to load, when one of them covers it (a folder holding no files yet does not list). Called at plan for a table whose files in S3 the destination would read where they lie; the error says why it can't, with the fix. +pub async fn list_reach(destination: &Destination, folder: Option<&str>) -> Result<(), String> { + let Destination::Databricks(databricks) = destination else { + return Ok(()); + }; + let locations = external_locations(databricks) + .await + .map_err(|error| format!("cannot list Databricks' external locations: {error}"))?; + let Some(folder) = folder else { + return Ok(()); + }; + let Some(url) = locations + .iter() + .find(|url| folder.starts_with(url.as_str())) + else { + return Ok(()); + }; + check_folder(databricks, folder).await.map_err(|error| { + let error = error.to_string(); + let first_line = error.lines().next().unwrap_or_default(); + format!( + "{folder} cannot be read by Databricks through the external location at {url}: {}. Fix: when Databricks refuses access, grant the service principal READ FILES on that external location", + first_line.trim_end_matches([':', ' ']) + ) + }) +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/mod.rs b/crates/sqlload/src/adapters/dispatch/main/route/mod.rs index 39b2a4702..562ba1684 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/mod.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/mod.rs @@ -1,5 +1,6 @@ pub mod choose_stage; pub mod direct_blocked; +pub mod list_reach; pub mod route_warning; pub mod sends_rows; pub mod stage_misses; diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs index e83c9cad0..fc200ae63 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs @@ -4,7 +4,7 @@ use crate::adapters::contract::models::{DirectBatch, DirectLoaded}; use crate::adapters::dispatch::models::Writer; use crate::loader::errors::LoaderError; -/// Load `batch` by having the destination read the source's files itself (Snowflake's `COPY` from an external stage, ClickHouse's `s3()`); `None` when SQLLoad must read the batch. +/// Load `batch` by having the destination read the source's files itself (Snowflake's `COPY` from an external stage, ClickHouse's `s3()`, Databricks' `COPY INTO` from S3); `None` when SQLLoad must read the batch. pub async fn load_files( writer: &mut Writer, batch: &DirectBatch<'_>, @@ -12,6 +12,7 @@ pub async fn load_files( match writer { Writer::Snowflake(snowflake) => snowflake.copy_files(batch).await, Writer::ClickHouse(clickhouse) => clickhouse.copy_files(batch).await, + Writer::Databricks(databricks) => databricks.copy_files(batch).await, #[cfg(feature = "duckdb")] Writer::DuckDb(duckdb) => duckdb.copy_files(batch).await, _ => Ok(None), diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs b/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs index 2e83ebbb2..cdda522b4 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs @@ -1,26 +1,6 @@ //! The DuckDB-only checks that decide whether DuckDB may read a files table's files itself. use crate::files_source::models::FileColumn; -use crate::store_paths::models::StoreKind; - -/// Why an S3 store points somewhere DuckDB would not read: an endpoint in the connection or in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows. -pub(crate) fn custom_endpoint( - store: &StoreKind, - variable: &dyn Fn(&str) -> Option, -) -> Option { - let from_env = ["AWS_ENDPOINT", "AWS_ENDPOINT_URL"] - .iter() - .any(|name| variable(name).is_some_and(|value| !value.is_empty())); - match store { - StoreKind::S3 { - endpoint: Some(_), .. - } => Some("the S3 connection sets a custom endpoint".to_owned()), - StoreKind::S3 { .. } if from_env => { - Some("AWS_ENDPOINT or AWS_ENDPOINT_URL sets a custom S3 endpoint".to_owned()) - } - _ => None, - } -} /// A file column whose name DuckDB's file reader uses itself, in any case: such a column would take the place of `file_index`, which picks each row's per-file values. pub(crate) fn reserved_name(column: &FileColumn) -> Option { diff --git a/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs b/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs index 3c75f3491..1aaa2388e 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs @@ -1,7 +1,8 @@ //! Whether DuckDB may load a files table's files itself. +use crate::adapters::contract::main::_custom_endpoint::custom_endpoint; use crate::adapters::contract::main::_files_direct_problem::files_direct_problem; -use crate::adapters::duckdb::_helpers::direct_checks::{custom_endpoint, reserved_name}; +use crate::adapters::duckdb::_helpers::direct_checks::reserved_name; use crate::adapters::duckdb::_helpers::motherduck_connection::environment_variable; use crate::files_source::models::{FilesRead, SourceFormat}; use crate::loader::models::TableSpec; diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs index 3d18e2a5b..23f9bcc66 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs @@ -1,4 +1,4 @@ -use crate::adapters::duckdb::_helpers::direct_checks::custom_endpoint; +use crate::adapters::contract::main::_custom_endpoint::custom_endpoint; use crate::adapters::duckdb::_helpers::pull_sql::{default_region, s3_secret}; use crate::adapters::duckdb::tests::test_types::{AwsVariablesTestCase, S3SecretTestCase}; use crate::store_paths::models::StoreKind; diff --git a/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs b/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs index 4e147cb27..6ccfdcae9 100644 --- a/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs +++ b/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs @@ -14,6 +14,7 @@ pub(in crate::adapters) fn writer_accepts(stage: Option<&StageRoute>) -> WriterA rows_into: None, reads_files_itself: Some(SNOWFLAKE), copies_files_to: Some(user_stage()), + must_pull: None, }; match stage { Some(StageRoute::External(stage)) => WriterAccepts { diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index d51ae4786..d5b1ccfd2 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -19,6 +19,7 @@ use crate::adapters::dispatch::main::dialect::source_dialect::source_dialect; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use crate::adapters::dispatch::main::route::choose_stage::choose_stage; use crate::adapters::dispatch::main::route::direct_blocked::direct_blocked; +use crate::adapters::dispatch::main::route::list_reach::list_reach; use crate::adapters::dispatch::main::route::route_warning::route_warning; use crate::adapters::dispatch::main::route::stage_misses::stage_misses; use crate::adapters::dispatch::main::route::unload_blocked::unload_blocked; @@ -894,7 +895,7 @@ async fn plan_table_once( .extend(truncated_nanoseconds(spec, live.as_deref())); let route = match schema.blocked { Some(_) => None, - None => match table_route(spec, files.as_ref(), stage.as_ref(), destination) { + None => match table_route(spec, files.as_ref(), stage.as_ref(), destination).await { Ok(route) => Some(route), Err(problem) => { schema = blocked_plan(vec![problem]); @@ -1204,14 +1205,22 @@ fn copy_warning(route: Option<&Route>) -> Option { )) } -/// The route a table's data takes into `destination`; `read` is a files table's planned read. -pub(crate) fn table_route( +/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 that nothing sends through SQLLoad, it first looks up where the destination reads files itself (Databricks' external locations). +pub(crate) async fn table_route( spec: &TableSpec, read: Option<&FilesRead>, stage: Option<&StageRoute>, destination: &Destination, ) -> Result { let reader = reader_offer(spec, read, stage); + let in_place = reader.needs_loader.is_none() && reader.direct_blocked.is_none(); + if let Some(files) = spec.files.as_ref().filter(|_| in_place) + && let StoreKind::S3 { .. } = files.location.store + { + let folder = files.location.to_string(); + let holds_files = read.is_some_and(|read| !read.files.is_empty()); + list_reach(destination, holds_files.then_some(folder.as_str())).await?; + } plan_route( &reader, &writer_accepts(destination, stage), diff --git a/crates/sqlload/src/routing/_helpers/files_route.rs b/crates/sqlload/src/routing/_helpers/files_route.rs index e59c5a550..f2cd52a33 100644 --- a/crates/sqlload/src/routing/_helpers/files_route.rs +++ b/crates/sqlload/src/routing/_helpers/files_route.rs @@ -49,7 +49,7 @@ pub(crate) fn warehouse_reads( } } -/// Files for a destination that does not read them itself: in place where it can, otherwise read by SQLLoad. +/// Files for a destination that does not read them itself: in place where it can, otherwise read by SQLLoad, except files it reads only itself ([`WriterAccepts::must_pull`]), which fail. pub(crate) fn other_reads( reader: &ReaderOffer, writer: &WriterAccepts, @@ -71,6 +71,16 @@ pub(crate) fn other_reads( overrides, &format!("{LOADER_READS_BECAUSE} {reason}"), ), + (false, None, Some(reason)) if must_pull(writer, files).is_some() => loader_reads( + files, + writer, + overrides, + &format!("{LOADER_READS_BECAUSE} {reason}"), + ), + (false, None, None) if must_pull(writer, files).is_some() => Err(RouteError::Uncovered { + location: files.store.to_string(), + reason: must_pull(writer, files).unwrap_or_default(), + }), (false, None, _) => loader_reads( files, writer, @@ -80,6 +90,17 @@ pub(crate) fn other_reads( } } +/// Why `writer` refuses `files` it cannot read where they lie, when it reads such files only itself. +fn must_pull(writer: &WriterAccepts, files: &FileSet) -> Option { + writer + .must_pull + .as_ref() + .filter(|(store, _)| { + files.store.is_within(store) && writer.pull_formats.contains(&files.format) + }) + .map(|(_, reason)| reason.clone()) +} + /// SQLLoad reads the files into Arrow and hands them over: the only route on which file data passes through SQLLoad. pub(crate) fn loader_reads( files: &FileSet, diff --git a/crates/sqlload/src/routing/errors.rs b/crates/sqlload/src/routing/errors.rs index e1988178f..71c8bc5b5 100644 --- a/crates/sqlload/src/routing/errors.rs +++ b/crates/sqlload/src/routing/errors.rs @@ -10,6 +10,8 @@ pub enum RouteError { warehouse: &'static str, location: String, }, + #[error("{location} {reason}")] + Uncovered { location: String, reason: String }, #[error("the destination cannot load these files exactly by reading them itself: {0}")] DirectRefused(String), #[error("stage = \"{wanted}\" was requested, but the destination has no {wanted} stage")] diff --git a/crates/sqlload/src/routing/models.rs b/crates/sqlload/src/routing/models.rs index 36b504233..c8310c402 100644 --- a/crates/sqlload/src/routing/models.rs +++ b/crates/sqlload/src/routing/models.rs @@ -104,6 +104,8 @@ pub struct WriterAccepts { pub reads_files_itself: Option<&'static str>, /// Where files the destination cannot reach are uploaded unchanged for `copy_to_internal_stage`. pub copies_files_to: Option, + /// Files of `pull_formats` in this store it reads only itself, with why one `pulls_from` doesn't cover is refused and the fix: they never go through SQLLoad, unless `ReaderOffer::needs_loader` or `direct_blocked` says why they must. + pub must_pull: Option<(Store, String)>, } /// How a warehouse source is read. diff --git a/crates/sqlload/src/routing/tests/helpers.rs b/crates/sqlload/src/routing/tests/helpers.rs index c423ba1e5..589da0f5d 100644 --- a/crates/sqlload/src/routing/tests/helpers.rs +++ b/crates/sqlload/src/routing/tests/helpers.rs @@ -60,6 +60,7 @@ pub(crate) fn warehouse_destination(staging: Vec) -> WriterAccepts { rows_into: None, reads_files_itself: Some("Snowflake"), copies_files_to: Some(user_stage()), + must_pull: None, } } @@ -73,6 +74,22 @@ pub(crate) fn clickhouse_destination() -> WriterAccepts { rows_into: None, reads_files_itself: None, copies_files_to: None, + must_pull: None, + } +} + +/// Databricks: Parquet files in its volume, which SQLLoad stages, or in S3 under its one external location (the lake), which it reads in place; Parquet in S3 elsewhere it refuses. +pub(crate) fn databricks_destination() -> WriterAccepts { + let volume = Store::InternalStage("/Volumes/main/raw/stage".to_owned()); + WriterAccepts { + pulls_from: vec![volume.clone(), object_store(LAKE)], + pull_formats: vec![FileFormat::Parquet], + staging: vec![volume], + must_pull: Some(( + object_store("s3://"), + "cannot be read by Databricks: no Unity Catalog external location covers it".to_owned(), + )), + ..WriterAccepts::default() } } @@ -96,6 +113,7 @@ pub(crate) fn local_destination() -> WriterAccepts { rows_into: None, reads_files_itself: None, copies_files_to: None, + must_pull: None, } } diff --git a/crates/sqlload/src/routing/tests/test_plan_route.rs b/crates/sqlload/src/routing/tests/test_plan_route.rs index f9be85617..6c95a2a23 100644 --- a/crates/sqlload/src/routing/tests/test_plan_route.rs +++ b/crates/sqlload/src/routing/tests/test_plan_route.rs @@ -1,11 +1,13 @@ use crate::routing::main::plan_route::plan_route; use crate::routing::models::{FileFormat, ReadVia, ReaderOffer, RouteOverrides, StagePreference}; use crate::routing::tests::helpers::{ - LAKE, clickhouse_destination, database_destination, duckdb_destination, files_source, - internal_stage, local_destination, object_store, streaming_source, user_stage, + LAKE, clickhouse_destination, database_destination, databricks_destination, duckdb_destination, + files_source, internal_stage, local_destination, object_store, streaming_source, user_stage, warehouse_destination, warehouse_source, }; -use crate::routing::tests::test_types::{PlanRouteErrorTestCase, PlanRouteTestCase}; +use crate::routing::tests::test_types::{ + PlanRouteErrorTestCase, PlanRouteReasonTestCase, PlanRouteTestCase, +}; const HMAC: &str = "email masked with hmac, whose key must never appear in SQL"; const SMALL: Option = Some(1024 * 1024); @@ -64,6 +66,43 @@ fn given_a_source_and_destination_when_planning_then_the_cheapest_route_is_chose "destination loads parquet files from s3://lake/scores/date=2026-11-04/", ], }, + PlanRouteTestCase { + description: "Parquet in S3 under Databricks' external location is copied where it lies", + reader: files_source("s3://lake/scores/", FileFormat::Parquet), + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_steps: &[ + "use parquet files in s3://lake/scores/", + "destination loads parquet files from s3://lake/scores/", + ], + }, + PlanRouteTestCase { + description: "CSV in S3 under Databricks' external location goes through SQLLoad into the volume", + reader: files_source("s3://lake/scores/", FileFormat::Csv), + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_steps: &[ + "use csv files in s3://lake/scores/", + "read csv into Arrow", + "write Parquet to internal stage /Volumes/main/raw/stage", + "destination loads parquet files from internal stage /Volumes/main/raw/stage", + ], + }, + PlanRouteTestCase { + description: "Parquet in S3 Databricks can't load exactly goes through SQLLoad, covered or not", + reader: ReaderOffer { + direct_blocked: Some("at is a time".to_owned()), + ..files_source("s3://other/scores/", FileFormat::Parquet) + }, + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_steps: &[ + "use parquet files in s3://other/scores/", + "read parquet into Arrow", + "write Parquet to internal stage /Volumes/main/raw/stage", + "destination loads parquet files from internal stage /Volumes/main/raw/stage", + ], + }, PlanRouteTestCase { description: "existing files the warehouse cannot reach are read and restaged only for hmac", reader: ReaderOffer { @@ -331,6 +370,13 @@ fn given_an_impossible_combination_when_planning_then_the_error_says_why() { overrides: RouteOverrides::default(), expected_message: "the destination cannot load these files exactly by reading them itself: half is Float16 in the files", }, + PlanRouteErrorTestCase { + description: "Parquet in S3 outside every external location fails Databricks, never going through SQLLoad", + reader: files_source("s3://other/events/", FileFormat::Parquet), + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_message: "s3://other/events/ cannot be read by Databricks: no Unity Catalog external location covers it", + }, PlanRouteErrorTestCase { description: "a source with nothing to offer", reader: crate::routing::models::ReaderOffer::default(), @@ -351,3 +397,30 @@ fn given_an_impossible_combination_when_planning_then_the_error_says_why() { ); } } + +#[test] +fn given_files_sqlload_must_read_when_planning_then_the_route_says_why() { + let test_cases = [PlanRouteReasonTestCase { + description: "Parquet in S3 Databricks can't load exactly, outside every external location", + reader: ReaderOffer { + direct_blocked: Some("at is a time".to_owned()), + ..files_source("s3://other/scores/", FileFormat::Parquet) + }, + writer: databricks_destination(), + expected_reason: "SQLLoad reads the files because at is a time", + }]; + for test_case in test_cases { + let reason = plan_route( + &test_case.reader, + &test_case.writer, + RouteOverrides::default(), + ) + .map(|route| route.reason) + .unwrap_or_default(); + assert!( + reason.starts_with(test_case.expected_reason), + "{}: {reason}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/routing/tests/test_types.rs b/crates/sqlload/src/routing/tests/test_types.rs index 708710423..b9b770392 100644 --- a/crates/sqlload/src/routing/tests/test_types.rs +++ b/crates/sqlload/src/routing/tests/test_types.rs @@ -16,3 +16,11 @@ pub(crate) struct PlanRouteErrorTestCase { pub(crate) overrides: RouteOverrides, pub(crate) expected_message: &'static str, } + +pub(crate) struct PlanRouteReasonTestCase { + pub(crate) description: &'static str, + pub(crate) reader: ReaderOffer, + pub(crate) writer: WriterAccepts, + /// How the route's reason starts. + pub(crate) expected_reason: &'static str, +} diff --git a/website/public/index.html b/website/public/index.html index f2078c4d4..b3791b3b0 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -170,7 +170,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 1c87b78f7..a785bdd2f 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -29,7 +29,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 |-------------|------------------|------------------|
 | Snowflake | `append`, `merge`, `delete_insert`, `replace` | Parquet staged on an external, user, table or named stage, then `COPY`; or Snowpipe Streaming, or `INSERT`, by `load_method`. Files under an external stage load with `COPY` directly. |
 | BigQuery (beta) | `append`, `merge`, `delete_insert`, `replace` | Arrow streamed through the Storage Write API into a staging table, then one query job per write. See [BigQuery](/docs/targets/bigquery/). |
-| Databricks (beta) | `append`, `merge`, `delete_insert`, `replace` | Parquet staged in a Unity Catalog volume, then `COPY INTO`; one statement per write. Files of a files source pass through SQLLoad. |
+| Databricks (beta) | `append`, `merge`, `delete_insert`, `replace` | Parquet in S3 copied where it lies through a Unity Catalog external location; everything else staged as Parquet in a volume, then `COPY INTO`; one statement per write. |
 | ClickHouse, one self-hosted server | `append`, `merge`, `delete_insert`, `replace` | Arrow over HTTP into a staging table, published by swapping in a whole table or whole partitions; Parquet in S3 read by ClickHouse with `s3()`. No clusters, replicated or distributed tables; ClickHouse Cloud is untested. See [ClickHouse](/docs/targets/clickhouse/). |
 | Postgres | `append`, `merge`, `delete_insert`, `replace` | Binary `COPY` into a temporary staging table, then one transaction per commit. See [Postgres](/docs/targets/postgres/). |
 | DuckDB | `append`, `merge`, `delete_insert`, `replace` | Arrow appends; Parquet files read by DuckDB with `read_parquet`. |
diff --git a/website/src/content/docs/docs/targets/databricks.mdx b/website/src/content/docs/docs/targets/databricks.mdx
index 6e922b145..efe8be369 100644
--- a/website/src/content/docs/docs/targets/databricks.mdx
+++ b/website/src/content/docs/docs/targets/databricks.mdx
@@ -1,6 +1,6 @@
 ---
 title: "Databricks"
-description: "Load into Databricks (beta), a SQL warehouse with Unity Catalog: Parquet staged in a volume, then COPY INTO, one statement per write."
+description: "Load into Databricks (beta), a SQL warehouse with Unity Catalog: Parquet in S3 copied where it lies through an external location, everything else staged in a volume, then COPY INTO, one statement per write."
 sidebar:
   badge:
     text: Beta
@@ -9,9 +9,9 @@ sidebar:
 
 This target is in beta: it is tested, but gets less everyday use than the other connectors, so expect rough edges, and please report them in [GitHub issues](https://github.com/sqlstack-io/sqlload/issues).
 
-A Databricks target loads into a Unity Catalog SQL warehouse. SQLLoad writes each batch as Parquet
-to a Unity Catalog volume and loads it with `COPY INTO`. That is the only route: there is no
-`INSERT` fallback.
+A Databricks target loads into a Unity Catalog SQL warehouse with `COPY INTO`: from Parquet files in
+S3 where they lie, through an external location, and from Parquet SQLLoad writes to a Unity Catalog
+volume for everything else. There is no `INSERT` fallback.
 
 ## Connection
 
@@ -52,8 +52,42 @@ Each batch is written as Parquet to `/sqlload///` and lo
 next load removes folders a failed run left once they are over a day old; a folder a running load
 is still filling is kept.
 
-Files of a files source pass through SQLLoad on the way to the volume. `copy_to_internal_stage` is
-for Snowflake and is refused here.
+### Parquet files in S3
+
+Parquet files of a files source in S3 load straight from the bucket, with no row passing through
+SQLLoad, when a Unity Catalog external location covers their folder. `plan` lists the external
+locations the service principal can see, and checks with `LIST` that it can read a folder holding
+files to load, which needs `READ FILES` on the location it reads through:
+
+```sql
+CREATE EXTERNAL LOCATION lake URL 's3://example-bucket/events/' WITH (STORAGE CREDENTIAL lake_reader);
+GRANT READ FILES ON EXTERNAL LOCATION lake TO ``;
+```
+
+```text
+route   Databricks COPYs directly from s3://example-bucket/events/ (nothing through the loader)
+```
+
+Each batch is one `COPY INTO` over its files, and each `COPY` is one transaction. A batch needs
+more than one only when it holds over 1,000 files (the most `FILES` takes, as one partition of a
+`delete_insert` on a partition cursor can) or files holding one column as different types, such as
+a 32-bit integer in one file and a 64-bit one in another, which Databricks won't merge in one
+`COPY`. Every strategy but `append` lands its batches in a staging or build table, so a batch
+publishes whole. An `append` copies straight into the table and commits each `COPY`; when a later
+one fails, or the rows Databricks reports inserting differ from what the files' footers hold, it
+deletes the rows this load wrote from the batch's files before the load fails. The columns SQLLoad
+adds (`_sqlload_source_file`, `_sqlload_source_version`, `_sqlload_source_row`, the load id and
+time) and partition values from the path are filled in the same statement.
+
+Parquet in S3 that no external location covers fails `plan`, naming the folder and the fix; such
+files never go through SQLLoad instead. Files go through SQLLoad, staged in the volume as below,
+when Databricks can't load them exactly by reading them itself, and `plan` says why, such as for
+CSV and JSON files, a table that masks columns, has checks or flattens objects, a column held in a way a cast
+wouldn't keep exactly (nanoseconds, a type other than the one it loads as), a time, a UUID or a
+list, and files on an S3-compatible endpoint (set on the connection or in `AWS_ENDPOINT` or
+`AWS_ENDPOINT_URL`), which no external location reaches. Files in a local
+folder or over SFTP always go through SQLLoad. `copy_to_internal_stage` is for Snowflake and is
+refused here.
 
 ## Write strategies
 
@@ -117,6 +151,12 @@ values checked against each source after every round.
 | BigQuery | not run | live | not run | live |
 | Files on an SFTP server (gzip CSV, Parquet) | live | not run | not run | refused at `plan` |
 
+Parquet in S3 copied through an external location was also run live with `append`, `replace` and
+`delete_insert` (a staging table), including files mixing 32-bit and 64-bit integers, each table
+compared row for row, lineage columns included, with the same files loaded through SQLLoad. A
+folder under an external location the service principal can't read failed `plan` with the grant
+to add.
+
 `merge` is refused for files sources on every target. The live rounds also covered incremental
 loads on a cursor, rows with a NULL cursor under `delete_insert`, an added column, widening a
 decimal in place, `contract diff`, `contract diff --source` and `contract generate --write`, and a
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index cc500e889..a1319a679 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -15,7 +15,7 @@ names the route for every table.
 | DuckDB | DuckDB reads them with `read_parquet` | Read by SQLLoad, appended as Arrow | Appended as Arrow |
 | MotherDuck (beta) | Read by SQLLoad, appended as Arrow | Read by SQLLoad, appended as Arrow | Appended as Arrow |
 | Snowflake | `COPY` from an external stage over the bucket | `COPY` from an external stage over the bucket | Written as Parquet to the `load_method`'s stage, then `COPY` (or streamed or inserted, if the `load_method` says so) |
-| Databricks (beta) | Staged by SQLLoad in the volume, then `COPY INTO` | Read by SQLLoad, staged as Parquet in the volume, then `COPY INTO` | Written as Parquet to the volume, then `COPY INTO` |
+| Databricks (beta) | In S3: `COPY INTO` where they lie, through an external location. In a folder: staged by SQLLoad in the volume, then `COPY INTO` | Read by SQLLoad, staged as Parquet in the volume, then `COPY INTO` | Written as Parquet to the volume, then `COPY INTO` |
 | BigQuery (beta) | Read by SQLLoad, streamed as Arrow into a staging table, then one query job | As for Parquet files | Streamed as Arrow into a staging table, then one query job |
 | ClickHouse | In S3: ClickHouse reads them with `s3()` into a staging table. In a folder: read by SQLLoad, sent as Arrow | Read by SQLLoad, sent as Arrow into a staging table | Sent as Arrow into a staging table, then swapped in by table or partition |
 | Postgres | Read by SQLLoad, then binary `COPY` into a temporary staging table | As for Parquet files | Binary `COPY` into a temporary staging table, then one transaction per commit |