From 7a2bb7e520303508995118ee874f5e86752717b9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:33:00 +0000 Subject: [PATCH 1/4] wip: Databricks COPYs Parquet files in S3 through a Unity Catalog external location; uncovered S3 Parquet fails plan with the fix --- .../databricks/_helpers/write/copy_select.rs | 32 ++-- .../databricks/_helpers/write/direct_load.rs | 145 ++++++++++++++++++ .../adapters/databricks/_helpers/write/mod.rs | 2 + .../databricks/_helpers/write/s3_copy.rs | 101 ++++++++++++ .../src/adapters/databricks/constants.rs | 10 ++ .../main/session/_connect_databricks.rs | 1 + .../databricks/main/stage/_direct_problem.rs | 37 +++++ .../main/stage/_external_locations.rs | 33 ++++ .../databricks/main/stage/_writer_accepts.rs | 15 +- .../src/adapters/databricks/main/stage/mod.rs | 2 + .../sqlload/src/adapters/databricks/models.rs | 2 + .../dispatch/main/route/direct_blocked.rs | 4 +- .../dispatch/main/route/list_reach.rs | 15 ++ .../src/adapters/dispatch/main/route/mod.rs | 1 + .../dispatch/main/write/load_files.rs | 3 +- .../snowflake/main/stage/_writer_accepts.rs | 1 + .../sqlload/src/loader/_helpers/table_plan.rs | 7 + .../src/routing/_helpers/files_route.rs | 17 +- crates/sqlload/src/routing/errors.rs | 2 + crates/sqlload/src/routing/models.rs | 2 + 20 files changed, 414 insertions(+), 18 deletions(-) create mode 100644 crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs create mode 100644 crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs create mode 100644 crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs create mode 100644 crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs create mode 100644 crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs index 5d2d686c2..4f6058082 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/copy_select.rs @@ -9,28 +9,18 @@ use crate::adapters::contract::types::SqlAdapter; use crate::adapters::databricks::constants::COPY_OPTIONS; use crate::adapters::databricks::models::DatabricksSql; use crate::column_types::models::ColumnType; +use crate::schema_plan::models::DesiredColumn; -/// `COPY INTO` the batch's table (the staging or build table, or the table itself) from `folder`, each column cast to the type it lands as: the table's own types straight into it or a build table, JSON parsed; text for JSON and UUIDs in a staging table. +/// `COPY INTO` the batch's table (the staging or build table, or the table itself) from `folder`, each column cast to the type it lands as (see [`typed_value`]). pub(crate) fn copy_statement(target: &WriteTarget<'_>, folder: &str) -> String { let adapter = DatabricksSql; let table = target.staging.unwrap_or(target.table); - let final_types = target.direct || target.builds; let selected: Vec = target .columns .iter() .map(|column| { let name = adapter.render_identifier(&column.name); - match (final_types, &column.column_type) { - (true, ColumnType::Json) => format!("parse_json({name}) AS {name}"), - (true, column_type) => format!( - "CAST({name} AS {}) AS {name}", - adapter.render_loader_logical_type(column_type).sql - ), - (false, column_type) => format!( - "CAST({name} AS {}) AS {name}", - adapter.render_staging_type(column_type) - ), - } + format!("{} AS {name}", typed_value(target, column, &name)) }) .collect(); format!( @@ -41,6 +31,22 @@ pub(crate) fn copy_statement(target: &WriteTarget<'_>, folder: &str) -> String { ) } +/// `value` as `column` lands in `target`'s batch table: in the table's own type straight into it or a build table, JSON parsed; in a staging table, text for JSON and UUIDs. +pub(crate) fn typed_value(target: &WriteTarget<'_>, column: &DesiredColumn, value: &str) -> String { + let adapter = DatabricksSql; + match (target.direct || target.builds, &column.column_type) { + (true, ColumnType::Json) => format!("parse_json({value})"), + (true, column_type) => format!( + "CAST({value} AS {})", + adapter.render_loader_logical_type(column_type).sql + ), + (false, column_type) => format!( + "CAST({value} AS {})", + adapter.render_staging_type(column_type) + ), + } +} + /// `batch` with its time-of-day columns as text, since Databricks reads no Parquet `TIME` and stores times as text. pub(crate) fn readable_batch(batch: RecordBatch) -> Result { let schema = batch.schema(); diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs new file mode 100644 index 000000000..97f868cfc --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs @@ -0,0 +1,145 @@ +//! A files batch loaded by Databricks `COPY`ing the source's own Parquet files from S3. + +use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem; +use crate::adapters::contract::main::_footer_rows::footer_rows; +use crate::adapters::contract::models::{DirectBatch, DirectLoaded}; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::s3_copy::{S3Batch, S3File, s3_copy_statement}; +use crate::adapters::databricks::main::stage::_direct_problem::direct_problem; +use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter}; +use crate::databricks_api::main::execute_statement::execute_statement; +use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN}; +use crate::files_source::main::normal_value::normal_value; +use crate::files_source::models::DataFile; +use crate::loader::errors::LoaderError; +use crate::schema_plan::models::DesiredColumn; + +impl DatabricksWriter { + /// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, in one `COPY INTO` (one transaction) into the staging or build table the load publishes from, or into the table itself. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose loaded rows differ from its footers' (an append into the table itself has then already committed them); files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location. + pub(in crate::adapters) async fn copy_files( + &mut self, + batch: &DirectBatch<'_>, + ) -> Result, LoaderError> { + let Some(source) = batch.spec.files.as_ref() else { + return Ok(None); + }; + let folder = source.location.to_string(); + let covered = self + .destination + .external_locations + .get() + .is_some_and(|locations| locations.iter().any(|url| folder.starts_with(url.as_str()))); + if !covered { + return Ok(None); + } + if let Some(problem) = + batch_direct_problem(batch, source, direct_problem(batch.spec, batch.read)) + { + return Err(LoaderError::Invalid(format!( + "Databricks cannot load this batch's files exactly by reading them itself, and they never go through SQLLoad instead: {problem}" + ))); + } + let target = &batch.target; + let (into, columns) = match (target.staging, target.builds) { + (Some(staging), false) => (staging, target.columns), + (staging, _) => (staging.unwrap_or(target.table), batch.write_columns), + }; + let mut files = Vec::with_capacity(batch.files.len()); + for data in batch.files { + files.push(s3_file(batch, data, columns)?); + } + let mut in_files: Vec<(String, String)> = Vec::new(); + for footer in batch.footers.parquet.values() { + for field in footer.schema().fields() { + let loaded = batch.spec.source_table.loaded_as(field.name()).to_owned(); + if !in_files.iter().any(|(name, _)| *name == loaded) { + in_files.push((loaded, field.name().clone())); + } + } + } + let statement = s3_copy_statement( + (&DatabricksSql.render_qualified_name(into), columns), + target, + &S3Batch { + folder, + files, + in_files, + }, + batch.loaded_at, + ); + let result = execute_statement(&self.destination.api, &statement).await?; + let inserted = result + .columns + .iter() + .position(|column| column == "num_inserted_rows") + .and_then(|index| { + result + .rows + .first()? + .get(index)? + .as_deref()? + .parse::() + .ok() + }); + let expected = footer_rows(batch.footers); + if inserted != Some(expected) { + return Err(LoaderError::Invalid(format!( + "Databricks loaded {} rows from this batch's files, but their footers hold {expected}", + inserted.map_or_else( + || "an unreported number of".to_owned(), + |rows| rows.to_string() + ) + ))); + } + Ok(Some(match target.staging { + Some(_) => DirectLoaded::Staged(expected), + None => DirectLoaded::Committed(expected), + })) + } +} + +/// The values SQLLoad gives every row of `data`: its name and version as the files source records them, and its partition values (constants among them), each converted to its column's type as SQLLoad's reader would, so a value that does not convert fails here rather than in Databricks' cast. +fn s3_file( + batch: &DirectBatch<'_>, + data: &DataFile, + columns: &[DesiredColumn], +) -> Result { + let Some(source) = batch.spec.files.as_ref() else { + return Err(LoaderError::Invalid("not a files table".to_owned())); + }; + let mut constants = vec![ + ( + SOURCE_FILE_COLUMN.to_owned(), + Some(format!("{}{}", source.location, data.file.relative_key)), + ), + (SOURCE_VERSION_COLUMN.to_owned(), Some(data.file.version())), + ]; + for (name, _) in &source.partition_types { + let value = data + .partitions + .iter() + .find(|(partition, _)| partition == name) + .map(|(_, value)| value.clone()); + let loaded = batch.spec.source_table.loaded_as(name); + let column_type = columns + .iter() + .find(|column| column.name == loaded) + .map(|column| &column.column_type); + let value = match (value, column_type) { + (Some(value), Some(column_type)) => { + Some(normal_value(&value, column_type).map_err(|reason| { + LoaderError::Invalid(format!( + "{}: {name} = {value:?} does not load as {column_type}: {reason}", + data.file.relative_key + )) + })?) + } + (value, _) => value, + }; + constants.push((loaded.to_owned(), value)); + } + Ok(S3File { + key: data.file.relative_key.clone(), + constants, + }) +} diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs index 256e51289..30bac9520 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod copy_select; +pub(crate) mod direct_load; +pub(crate) mod s3_copy; pub(crate) mod table_writer; diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs new file mode 100644 index 000000000..4425006b1 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs @@ -0,0 +1,101 @@ +//! The `COPY INTO` that has Databricks read a files batch's Parquet files where they lie in S3. + +use crate::adapters::contract::models::WriteTarget; +use crate::adapters::contract::types::SqlAdapter; +use crate::adapters::databricks::_helpers::write::copy_select::typed_value; +use crate::adapters::databricks::constants::S3_COPY_OPTIONS; +use crate::adapters::databricks::models::DatabricksSql; +use crate::files_source::constants::SOURCE_ROW_COLUMN; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::MetadataRole; +use crate::schema_plan::models::DesiredColumn; + +/// The path `COPY INTO` reads each row from, as SQLLoad names it: `_metadata.file_path` is percent-encoded, except for `+`, which `url_decode` alone would read as a space. +const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))"; + +/// One batch read from S3: the folder `COPY INTO` reads (`s3://bucket/prefix/`), and for each file its key below it and the values SQLLoad gives every row of it (`None` for NULL), by column. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct S3Batch { + pub folder: String, + pub files: Vec, + /// Each table column the batch's files hold, with its name in them, which `naming`, `rename` or numbering may have changed. + pub in_files: Vec<(String, String)>, +} + +/// One file of an [`S3Batch`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct S3File { + pub key: String, + pub constants: Vec<(String, Option)>, +} + +/// `COPY INTO into` every one of `columns`, typed as `target` lands them, from `batch`'s files: a column takes the file's own value when SQLLoad gives it one (picked by the file's path, and a row from any other path fails the statement), the files' column when they hold it, and NULL otherwise; `_sqlload_source_row` numbers each file's rows from 1 in stored order; the load id and load time are the batch's. +pub(crate) fn s3_copy_statement( + (into, columns): (&str, &[DesiredColumn]), + target: &WriteTarget<'_>, + batch: &S3Batch, + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let values: Vec = columns + .iter() + .map(|column| { + let given: Vec<(&S3File, Option<&String>)> = batch + .files + .iter() + .filter_map(|file| { + let (_, value) = file + .constants + .iter() + .find(|(name, _)| *name == column.name)?; + Some((file, value.as_ref())) + }) + .collect(); + let value = match metadata_role(target.compat, &column.name) { + _ if !given.is_empty() => per_file(batch, &given), + Some(MetadataRole::LoadId) => literal(target.load_id), + Some(MetadataRole::LoadedAt) => literal(loaded_at), + _ if column.name == SOURCE_ROW_COLUMN => "_metadata.row_index + 1".to_owned(), + _ => batch + .in_files + .iter() + .find(|(name, _)| *name == column.name) + .map_or_else( + || "NULL".to_owned(), + |(_, held)| DatabricksSql.render_identifier(held), + ), + }; + format!( + "{} AS {}", + typed_value(target, column, &value), + DatabricksSql.render_identifier(&column.name) + ) + }) + .collect(); + let keys: Vec = batch.files.iter().map(|file| literal(&file.key)).collect(); + format!( + "COPY INTO {into} FROM (SELECT {} FROM {}) FILEFORMAT = PARQUET FILES = ({}) {S3_COPY_OPTIONS}", + values.join(", "), + literal(&batch.folder), + keys.join(", ") + ) +} + +/// The value each of `given`'s files gives a column, picked by the path a row was read from; a path not in the batch fails the statement rather than load a row without it. +fn per_file(batch: &S3Batch, given: &[(&S3File, Option<&String>)]) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let picks: Vec = given + .iter() + .map(|(file, value)| { + format!( + "WHEN {} THEN {}", + literal(&format!("{}{}", batch.folder, file.key)), + value.map_or_else(|| "NULL".to_owned(), |value| literal(value)) + ) + }) + .collect(); + format!( + "CASE {FILE_PATH} {} ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END", + picks.join(" ") + ) +} diff --git a/crates/sqlload/src/adapters/databricks/constants.rs b/crates/sqlload/src/adapters/databricks/constants.rs index b6f1565a9..ad90c52bc 100644 --- a/crates/sqlload/src/adapters/databricks/constants.rs +++ b/crates/sqlload/src/adapters/databricks/constants.rs @@ -22,5 +22,15 @@ pub const TRANSACTION_STATEMENTS: &[&str] = &["BEGIN", "COMMIT", "ROLLBACK"]; /// How staged Parquet files are read: every file again, even one a `COPY INTO` has loaded before. pub const COPY_OPTIONS: &str = "FILEFORMAT = PARQUET COPY_OPTIONS ('force' = 'true')"; +/// How a batch's files are read where they lie in S3, after its `FILES` list: every column any of them holds (one lacking it gives NULL), every file again, even one a `COPY INTO` has loaded before. +pub const S3_COPY_OPTIONS: &str = + "FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')"; + +/// The external locations the service principal can see. +pub const LIST_LOCATIONS: &str = "SHOW EXTERNAL LOCATIONS"; + +/// Why `plan` refuses Parquet files in S3 that no external location covers, after their folder. +pub const OUTSIDE_LOCATIONS: &str = "cannot be read by Databricks: no Unity Catalog external location covers it, and Parquet files in S3 never go through SQLLoad into Databricks. Fix: create an external location whose URL holds the folder (CREATE EXTERNAL LOCATION ... URL 's3://bucket/folder/' WITH (STORAGE CREDENTIAL ...)), and grant the service principal READ FILES on it"; + /// Shown for `copy_to_internal_stage` on a Databricks target. pub const NO_INTERNAL_STAGE_COPY: &str = "copy_to_internal_stage is for Snowflake targets; Databricks loads the files through its staging volume"; diff --git a/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs b/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs index e01755732..ee7048b0f 100644 --- a/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs +++ b/crates/sqlload/src/adapters/databricks/main/session/_connect_databricks.rs @@ -41,5 +41,6 @@ pub(in crate::adapters) fn connect_databricks( catalog: catalog.unwrap_or(own_catalog).to_owned(), volume: volume_path(volume), ddl: Arc::default(), + external_locations: Arc::default(), }) } diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs b/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs new file mode 100644 index 000000000..2d2db6924 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs @@ -0,0 +1,37 @@ +//! Whether Databricks may load a files table's files itself. + +use crate::adapters::contract::main::_files_direct_problem::files_direct_problem; +use crate::column_types::models::ColumnType; +use crate::files_source::models::{FilesRead, SourceFormat}; +use crate::loader::models::TableSpec; +use crate::store_paths::models::StoreKind; + +/// Why Databricks cannot load the planned `read` of a files table in S3 exactly by `COPY`ing its Parquet files where they lie, so SQLLoad reads them into the staging volume instead; `None` when it can, or when the files aren't in S3. Every column must load as the type the files hold it as, so Databricks' casts never differ from SQLLoad's; an external location reaches only AWS S3, not an S3-compatible endpoint; masks need SQLLoad's reader; and Databricks reads a Parquet time as text in its own format, a UUID as bytes, and lists as its own arrays. +pub(in crate::adapters) fn direct_problem(spec: &TableSpec, read: &FilesRead) -> Option { + let files = spec.files.as_ref()?; + let StoreKind::S3 { endpoint, .. } = &files.location.store else { + return None; + }; + let problems = [ + (files.encoding.format != SourceFormat::Parquet) + .then(|| "only Parquet files are read by Databricks itself".to_owned()), + endpoint.as_ref().map(|endpoint| { + format!("the files are on {endpoint}, which no Unity Catalog external location reaches") + }), + (!spec.masked.columns.is_empty()).then(|| "the table masks columns".to_owned()), + files_direct_problem(files, read), + read.columns.iter().find_map(|column| { + let held = match &column.column_type { + ColumnType::Time => { + "a time, which Databricks reads from Parquet as text in its own format" + } + ColumnType::Uuid => "a UUID, which Databricks reads from Parquet as bytes", + ColumnType::Array(_) => "a list, which Databricks reads as its own array", + ColumnType::Native(_) => "a type only SQLLoad's reader converts", + _ => return None, + }; + Some(format!("{} is {held}", column.name)) + }), + ]; + problems.into_iter().flatten().next() +} diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs new file mode 100644 index 000000000..60ca1ed5a --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs @@ -0,0 +1,33 @@ +//! The Unity Catalog external locations a Databricks destination reads files through. + +use crate::adapters::databricks::constants::LIST_LOCATIONS; +use crate::adapters::databricks::models::DatabricksDestination; +use crate::databricks_api::main::execute_statement::execute_statement; +use crate::http::errors::ApiError; + +/// The URLs of the external locations `destination`'s service principal can see, each ending in `/` so that a folder only covers what lies below it; listed once per run. +pub(in crate::adapters) async fn external_locations( + destination: &DatabricksDestination, +) -> Result<&[String], ApiError> { + let listed = destination + .external_locations + .get_or_try_init(|| async { + let rows = execute_statement(&destination.api, LIST_LOCATIONS).await?; + let url = rows + .columns + .iter() + .position(|column| column == "url") + .ok_or_else(|| { + ApiError::Protocol(format!("{LIST_LOCATIONS} returned no url column")) + })?; + Ok::, ApiError>( + rows.rows + .iter() + .filter_map(|row| row.get(url).cloned().flatten()) + .map(|url| format!("{}/", url.trim_end_matches('/'))) + .collect(), + ) + }) + .await?; + Ok(listed) +} diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs b/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs index dc3dca5fa..43c8e0317 100644 --- a/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs +++ b/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs @@ -1,16 +1,27 @@ //! What Databricks takes. +use crate::adapters::databricks::constants::OUTSIDE_LOCATIONS; use crate::adapters::databricks::models::DatabricksDestination; use crate::routing::models::{FileFormat, Store, WriterAccepts}; -/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads. +/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads; and Parquet files in S3 under one of its external locations, which `COPY INTO` reads where they lie. Parquet files in S3 no external location covers are refused, never read by SQLLoad. pub(in crate::adapters) fn writer_accepts(destination: &DatabricksDestination) -> WriterAccepts { let volume = Store::InternalStage(destination.volume.clone()); + let locations = destination + .external_locations + .get() + .into_iter() + .flatten() + .map(|url| Store::ObjectStore(url.clone())); WriterAccepts { arrow: false, - pulls_from: vec![volume.clone()], + pulls_from: std::iter::once(volume.clone()).chain(locations).collect(), pull_formats: vec![FileFormat::Parquet], staging: vec![volume], + must_pull: Some(( + Store::ObjectStore("s3://".to_owned()), + OUTSIDE_LOCATIONS.to_owned(), + )), ..WriterAccepts::default() } } diff --git a/crates/sqlload/src/adapters/databricks/main/stage/mod.rs b/crates/sqlload/src/adapters/databricks/main/stage/mod.rs index 548796b84..51e3be694 100644 --- a/crates/sqlload/src/adapters/databricks/main/stage/mod.rs +++ b/crates/sqlload/src/adapters/databricks/main/stage/mod.rs @@ -1,4 +1,6 @@ pub(in crate::adapters) mod _check_volume; +pub(in crate::adapters) mod _direct_problem; +pub(in crate::adapters) mod _external_locations; pub(in crate::adapters) mod _remove_folder; pub(in crate::adapters) mod _remove_stale; pub(in crate::adapters) mod _writer_accepts; diff --git a/crates/sqlload/src/adapters/databricks/models.rs b/crates/sqlload/src/adapters/databricks/models.rs index 0d4761af2..73e10675f 100644 --- a/crates/sqlload/src/adapters/databricks/models.rs +++ b/crates/sqlload/src/adapters/databricks/models.rs @@ -17,6 +17,8 @@ pub struct DatabricksDestination { pub volume: String, /// Held while a load's tables change the schema, one at a time. pub ddl: Arc>, + /// The URLs (each ending in `/`) of the Unity Catalog external locations it reads files through, listed once, at plan, when a table's files are in S3. + pub external_locations: Arc>>, } /// One table's write session on Databricks: Parquet parts uploaded to the volume, then loaded by `COPY INTO`. diff --git a/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs index 90298b3f5..f2bc3fc70 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs @@ -4,6 +4,7 @@ use crate::adapters::bigquery::constants::NO_INTERNAL_STAGE_COPY as BIGQUERY_NO_ use crate::adapters::clickhouse::constants::NO_INTERNAL_STAGE_COPY as CLICKHOUSE_NO_INTERNAL_STAGE_COPY; use crate::adapters::clickhouse::main::plan::_direct_problem::direct_problem as clickhouse_direct_problem; use crate::adapters::databricks::constants::NO_INTERNAL_STAGE_COPY; +use crate::adapters::databricks::main::stage::_direct_problem::direct_problem as databricks_direct_problem; #[cfg(feature = "duckdb")] use crate::adapters::duckdb::main::_direct_problem::direct_problem; use crate::adapters::postgres::constants::NO_INTERNAL_STAGE_COPY as POSTGRES_NO_INTERNAL_STAGE_COPY; @@ -14,7 +15,7 @@ use crate::column_types::models::Warehouse; use crate::files_source::models::FilesRead; use crate::loader::models::TableSpec; -/// Why the destination cannot load the planned `read` of a files table exactly from its files: for DuckDB, by reading them itself wherever they are; for ClickHouse, by reading them from S3; for Snowflake, under the external `stage` that covers them, or copied unchanged to an internal stage (`copy_to_internal_stage`); `None` when it can, or when neither applies. +/// Why the destination cannot load the planned `read` of a files table exactly from its files: for DuckDB, by reading them itself wherever they are; for ClickHouse, by reading them from S3; for Databricks, by `COPY`ing them from S3 through an external location; for Snowflake, under the external `stage` that covers them, or copied unchanged to an internal stage (`copy_to_internal_stage`); `None` when it can, or when neither applies. pub fn direct_blocked( spec: &TableSpec, read: &FilesRead, @@ -32,6 +33,7 @@ pub fn direct_blocked( return match (files.copy_to_internal_stage, spec.warehouse) { (true, _) => Some(refused.to_owned()), (false, Warehouse::ClickHouse) => clickhouse_direct_problem(spec, read), + (false, Warehouse::Databricks) => databricks_direct_problem(spec, read), (false, _) => None, }; } diff --git a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs new file mode 100644 index 000000000..8f69542e4 --- /dev/null +++ b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs @@ -0,0 +1,15 @@ +//! Where a destination reads files itself, beyond its own stores. + +use crate::adapters::databricks::main::stage::_external_locations::external_locations; +use crate::adapters::dispatch::models::Destination; + +/// Look up, once per run, where `destination` reads files itself beyond its own stores, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers: Databricks' Unity Catalog external locations. Called at plan for a table whose files are in S3; the error says why they can't be listed. +pub async fn list_reach(destination: &Destination) -> Result<(), String> { + let Destination::Databricks(databricks) = destination else { + return Ok(()); + }; + external_locations(databricks) + .await + .map(|_| ()) + .map_err(|error| format!("cannot list Databricks' external locations: {error}")) +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/mod.rs b/crates/sqlload/src/adapters/dispatch/main/route/mod.rs index 39b2a4702..562ba1684 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/mod.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/mod.rs @@ -1,5 +1,6 @@ pub mod choose_stage; pub mod direct_blocked; +pub mod list_reach; pub mod route_warning; pub mod sends_rows; pub mod stage_misses; diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs index e83c9cad0..fc200ae63 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs @@ -4,7 +4,7 @@ use crate::adapters::contract::models::{DirectBatch, DirectLoaded}; use crate::adapters::dispatch::models::Writer; use crate::loader::errors::LoaderError; -/// Load `batch` by having the destination read the source's files itself (Snowflake's `COPY` from an external stage, ClickHouse's `s3()`); `None` when SQLLoad must read the batch. +/// Load `batch` by having the destination read the source's files itself (Snowflake's `COPY` from an external stage, ClickHouse's `s3()`, Databricks' `COPY INTO` from S3); `None` when SQLLoad must read the batch. pub async fn load_files( writer: &mut Writer, batch: &DirectBatch<'_>, @@ -12,6 +12,7 @@ pub async fn load_files( match writer { Writer::Snowflake(snowflake) => snowflake.copy_files(batch).await, Writer::ClickHouse(clickhouse) => clickhouse.copy_files(batch).await, + Writer::Databricks(databricks) => databricks.copy_files(batch).await, #[cfg(feature = "duckdb")] Writer::DuckDb(duckdb) => duckdb.copy_files(batch).await, _ => Ok(None), diff --git a/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs b/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs index 4e147cb27..6ccfdcae9 100644 --- a/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs +++ b/crates/sqlload/src/adapters/snowflake/main/stage/_writer_accepts.rs @@ -14,6 +14,7 @@ pub(in crate::adapters) fn writer_accepts(stage: Option<&StageRoute>) -> WriterA rows_into: None, reads_files_itself: Some(SNOWFLAKE), copies_files_to: Some(user_stage()), + must_pull: None, }; match stage { Some(StageRoute::External(stage)) => WriterAccepts { diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index c8a357d47..2e0eceee0 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -19,6 +19,7 @@ use crate::adapters::dispatch::main::dialect::source_dialect::source_dialect; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use crate::adapters::dispatch::main::route::choose_stage::choose_stage; use crate::adapters::dispatch::main::route::direct_blocked::direct_blocked; +use crate::adapters::dispatch::main::route::list_reach::list_reach; use crate::adapters::dispatch::main::route::route_warning::route_warning; use crate::adapters::dispatch::main::route::stage_misses::stage_misses; use crate::adapters::dispatch::main::route::unload_blocked::unload_blocked; @@ -892,6 +893,12 @@ async fn plan_table_once( schema .warnings .extend(truncated_nanoseconds(spec, live.as_deref())); + let in_s3 = spec.files.as_ref().map(|files| &files.location.store); + if let (None, Some(StoreKind::S3 { .. })) = (&schema.blocked, in_s3) + && let Err(problem) = list_reach(destination).await + { + schema = blocked_plan(vec![problem]); + } let route = match schema.blocked { Some(_) => None, None => match table_route(spec, files.as_ref(), stage.as_ref(), destination) { diff --git a/crates/sqlload/src/routing/_helpers/files_route.rs b/crates/sqlload/src/routing/_helpers/files_route.rs index e59c5a550..11efe60ad 100644 --- a/crates/sqlload/src/routing/_helpers/files_route.rs +++ b/crates/sqlload/src/routing/_helpers/files_route.rs @@ -49,7 +49,7 @@ pub(crate) fn warehouse_reads( } } -/// Files for a destination that does not read them itself: in place where it can, otherwise read by SQLLoad. +/// Files for a destination that does not read them itself: in place where it can, otherwise read by SQLLoad, except files it reads only itself ([`WriterAccepts::must_pull`]), which fail. pub(crate) fn other_reads( reader: &ReaderOffer, writer: &WriterAccepts, @@ -71,6 +71,10 @@ pub(crate) fn other_reads( overrides, &format!("{LOADER_READS_BECAUSE} {reason}"), ), + (false, None, None) if must_pull(writer, files).is_some() => Err(RouteError::Uncovered { + location: files.store.to_string(), + reason: must_pull(writer, files).unwrap_or_default(), + }), (false, None, _) => loader_reads( files, writer, @@ -80,6 +84,17 @@ pub(crate) fn other_reads( } } +/// Why `writer` refuses `files` it cannot read where they lie, when it reads such files only itself. +fn must_pull(writer: &WriterAccepts, files: &FileSet) -> Option { + writer + .must_pull + .as_ref() + .filter(|(store, _)| { + files.store.is_within(store) && writer.pull_formats.contains(&files.format) + }) + .map(|(_, reason)| reason.clone()) +} + /// SQLLoad reads the files into Arrow and hands them over: the only route on which file data passes through SQLLoad. pub(crate) fn loader_reads( files: &FileSet, diff --git a/crates/sqlload/src/routing/errors.rs b/crates/sqlload/src/routing/errors.rs index e1988178f..71c8bc5b5 100644 --- a/crates/sqlload/src/routing/errors.rs +++ b/crates/sqlload/src/routing/errors.rs @@ -10,6 +10,8 @@ pub enum RouteError { warehouse: &'static str, location: String, }, + #[error("{location} {reason}")] + Uncovered { location: String, reason: String }, #[error("the destination cannot load these files exactly by reading them itself: {0}")] DirectRefused(String), #[error("stage = \"{wanted}\" was requested, but the destination has no {wanted} stage")] diff --git a/crates/sqlload/src/routing/models.rs b/crates/sqlload/src/routing/models.rs index de5b766f3..bfb1df75e 100644 --- a/crates/sqlload/src/routing/models.rs +++ b/crates/sqlload/src/routing/models.rs @@ -102,6 +102,8 @@ pub struct WriterAccepts { pub reads_files_itself: Option<&'static str>, /// Where files the destination cannot reach are uploaded unchanged for `copy_to_internal_stage`. pub copies_files_to: Option, + /// Files of `pull_formats` in this store it reads only itself, with why one `pulls_from` doesn't cover is refused and the fix: they never go through SQLLoad, unless `ReaderOffer::needs_loader` or `direct_blocked` says why they must. + pub must_pull: Option<(Store, String)>, } /// How a warehouse source is read. From 4851863615fa64556ebf73a949789ebb18754fd2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:46:31 +0000 Subject: [PATCH 2/4] wip: Databricks S3 route tests, fensu fixes, docs --- .../databricks/_helpers/write/direct_load.rs | 45 ++++++----- .../databricks/_helpers/write/s3_copy.rs | 72 +++++++++--------- .../src/adapters/databricks/constants.rs | 6 ++ .../main/stage/_external_locations.rs | 4 +- .../sqlload/src/adapters/databricks/tests.rs | 2 + .../src/adapters/databricks/tests/helpers.rs | 74 +++++++++++++++++++ .../adapters/databricks/tests/test_s3_copy.rs | 26 +++++++ .../sqlload/src/loader/_helpers/table_plan.rs | 15 ++-- crates/sqlload/src/routing/tests/helpers.rs | 18 +++++ .../src/routing/tests/test_plan_route.rs | 48 +++++++++++- website/public/index.html | 2 +- .../content/docs/docs/supported-systems.mdx | 2 +- .../content/docs/docs/targets/databricks.mdx | 42 +++++++++-- .../src/content/docs/docs/targets/index.mdx | 2 +- 14 files changed, 281 insertions(+), 77 deletions(-) create mode 100644 crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs index 97f868cfc..39a451ce7 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs @@ -5,6 +5,7 @@ use crate::adapters::contract::main::_footer_rows::footer_rows; use crate::adapters::contract::models::{DirectBatch, DirectLoaded}; use crate::adapters::contract::types::SqlAdapter; use crate::adapters::databricks::_helpers::write::s3_copy::{S3Batch, S3File, s3_copy_statement}; +use crate::adapters::databricks::constants::INSERTED_ROWS_COLUMN; use crate::adapters::databricks::main::stage::_direct_problem::direct_problem; use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter}; use crate::databricks_api::main::execute_statement::execute_statement; @@ -13,9 +14,10 @@ use crate::files_source::main::normal_value::normal_value; use crate::files_source::models::DataFile; use crate::loader::errors::LoaderError; use crate::schema_plan::models::DesiredColumn; +use crate::sql_api::models::SqlRows; impl DatabricksWriter { - /// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, in one `COPY INTO` (one transaction) into the staging or build table the load publishes from, or into the table itself. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose loaded rows differ from its footers' (an append into the table itself has then already committed them); files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location. + /// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, in one `COPY INTO` (one transaction) into the staging or build table the load publishes from, or into the table itself. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose inserted rows, as `COPY INTO` reports them, differ from its footers' (an append into the table itself has then already committed them); files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location. pub(in crate::adapters) async fn copy_files( &mut self, batch: &DirectBatch<'_>, @@ -68,27 +70,11 @@ impl DatabricksWriter { batch.loaded_at, ); let result = execute_statement(&self.destination.api, &statement).await?; - let inserted = result - .columns - .iter() - .position(|column| column == "num_inserted_rows") - .and_then(|index| { - result - .rows - .first()? - .get(index)? - .as_deref()? - .parse::() - .ok() - }); + let inserted = inserted_rows(&result)?; let expected = footer_rows(batch.footers); - if inserted != Some(expected) { + if inserted != expected { return Err(LoaderError::Invalid(format!( - "Databricks loaded {} rows from this batch's files, but their footers hold {expected}", - inserted.map_or_else( - || "an unreported number of".to_owned(), - |rows| rows.to_string() - ) + "Databricks loaded {inserted} rows from this batch's files, but their footers hold {expected}" ))); } Ok(Some(match target.staging { @@ -98,6 +84,25 @@ impl DatabricksWriter { } } +/// The rows a `COPY INTO`'s `result` reports it inserted. +fn inserted_rows(result: &SqlRows) -> Result { + let reported = result + .columns + .iter() + .position(|column| column == INSERTED_ROWS_COLUMN) + .and_then(|index| result.rows.first()?.get(index)?.clone()); + let Some(text) = reported else { + return Err(LoaderError::Invalid(format!( + "COPY INTO reported no {INSERTED_ROWS_COLUMN}" + ))); + }; + text.parse::().map_err(|error| { + LoaderError::Invalid(format!( + "COPY INTO reported {INSERTED_ROWS_COLUMN} = {text:?}: {error}" + )) + }) +} + /// The values SQLLoad gives every row of `data`: its name and version as the files source records them, and its partition values (constants among them), each converted to its column's type as SQLLoad's reader would, so a value that does not convert fails here rather than in Databricks' cast. fn s3_file( batch: &DirectBatch<'_>, diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs index 4425006b1..71b77b37b 100644 --- a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs +++ b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs @@ -37,41 +37,18 @@ pub(crate) fn s3_copy_statement( loaded_at: &str, ) -> String { let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); - let values: Vec = columns - .iter() - .map(|column| { - let given: Vec<(&S3File, Option<&String>)> = batch - .files - .iter() - .filter_map(|file| { - let (_, value) = file - .constants - .iter() - .find(|(name, _)| *name == column.name)?; - Some((file, value.as_ref())) - }) - .collect(); - let value = match metadata_role(target.compat, &column.name) { - _ if !given.is_empty() => per_file(batch, &given), - Some(MetadataRole::LoadId) => literal(target.load_id), - Some(MetadataRole::LoadedAt) => literal(loaded_at), - _ if column.name == SOURCE_ROW_COLUMN => "_metadata.row_index + 1".to_owned(), - _ => batch - .in_files - .iter() - .find(|(name, _)| *name == column.name) - .map_or_else( - || "NULL".to_owned(), - |(_, held)| DatabricksSql.render_identifier(held), - ), - }; - format!( - "{} AS {}", - typed_value(target, column, &value), - DatabricksSql.render_identifier(&column.name) - ) - }) - .collect(); + let mut values: Vec = Vec::with_capacity(columns.len()); + for column in columns { + values.push(format!( + "{} AS {}", + typed_value( + target, + column, + &column_value(target, batch, column, loaded_at) + ), + DatabricksSql.render_identifier(&column.name) + )); + } let keys: Vec = batch.files.iter().map(|file| literal(&file.key)).collect(); format!( "COPY INTO {into} FROM (SELECT {} FROM {}) FILEFORMAT = PARQUET FILES = ({}) {S3_COPY_OPTIONS}", @@ -81,6 +58,31 @@ pub(crate) fn s3_copy_statement( ) } +/// Where `column`'s value comes from in `batch`, before it is typed: the value SQLLoad gives each file, the load id or load time, the row's position in its file, the files' own column, or NULL when no file holds it. +fn column_value( + target: &WriteTarget<'_>, + batch: &S3Batch, + column: &DesiredColumn, + loaded_at: &str, +) -> String { + let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); + let mut given: Vec<(&S3File, Option<&String>)> = Vec::new(); + for file in &batch.files { + if let Some((_, value)) = file.constants.iter().find(|(name, _)| *name == column.name) { + given.push((file, value.as_ref())); + } + } + let held = batch.in_files.iter().find(|(name, _)| *name == column.name); + match (metadata_role(target.compat, &column.name), held) { + _ if !given.is_empty() => per_file(batch, &given), + (Some(MetadataRole::LoadId), _) => literal(target.load_id), + (Some(MetadataRole::LoadedAt), _) => literal(loaded_at), + _ if column.name == SOURCE_ROW_COLUMN => "_metadata.row_index + 1".to_owned(), + (_, Some((_, held))) => DatabricksSql.render_exact_identifier(held), + (_, None) => "NULL".to_owned(), + } +} + /// The value each of `given`'s files gives a column, picked by the path a row was read from; a path not in the batch fails the statement rather than load a row without it. fn per_file(batch: &S3Batch, given: &[(&S3File, Option<&String>)]) -> String { let literal = |text: &str| DatabricksSql.render_loader_value_literal(text); diff --git a/crates/sqlload/src/adapters/databricks/constants.rs b/crates/sqlload/src/adapters/databricks/constants.rs index ad90c52bc..c71d8ee47 100644 --- a/crates/sqlload/src/adapters/databricks/constants.rs +++ b/crates/sqlload/src/adapters/databricks/constants.rs @@ -29,6 +29,12 @@ pub const S3_COPY_OPTIONS: &str = /// The external locations the service principal can see. pub const LIST_LOCATIONS: &str = "SHOW EXTERNAL LOCATIONS"; +/// The column of [`LIST_LOCATIONS`] holding each location's URL. +pub const LOCATION_URL_COLUMN: &str = "url"; + +/// The column of a `COPY INTO`'s result holding the rows it inserted. +pub const INSERTED_ROWS_COLUMN: &str = "num_inserted_rows"; + /// Why `plan` refuses Parquet files in S3 that no external location covers, after their folder. pub const OUTSIDE_LOCATIONS: &str = "cannot be read by Databricks: no Unity Catalog external location covers it, and Parquet files in S3 never go through SQLLoad into Databricks. Fix: create an external location whose URL holds the folder (CREATE EXTERNAL LOCATION ... URL 's3://bucket/folder/' WITH (STORAGE CREDENTIAL ...)), and grant the service principal READ FILES on it"; diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs index 60ca1ed5a..ebe837c88 100644 --- a/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs +++ b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs @@ -1,6 +1,6 @@ //! The Unity Catalog external locations a Databricks destination reads files through. -use crate::adapters::databricks::constants::LIST_LOCATIONS; +use crate::adapters::databricks::constants::{LIST_LOCATIONS, LOCATION_URL_COLUMN}; use crate::adapters::databricks::models::DatabricksDestination; use crate::databricks_api::main::execute_statement::execute_statement; use crate::http::errors::ApiError; @@ -16,7 +16,7 @@ pub(in crate::adapters) async fn external_locations( let url = rows .columns .iter() - .position(|column| column == "url") + .position(|column| column == LOCATION_URL_COLUMN) .ok_or_else(|| { ApiError::Protocol(format!("{LIST_LOCATIONS} returned no url column")) })?; diff --git a/crates/sqlload/src/adapters/databricks/tests.rs b/crates/sqlload/src/adapters/databricks/tests.rs index fb552a6b6..57161de5d 100644 --- a/crates/sqlload/src/adapters/databricks/tests.rs +++ b/crates/sqlload/src/adapters/databricks/tests.rs @@ -2,6 +2,8 @@ mod dialect; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_s3_copy.rs"] +mod s3_copy; #[path = "tests/test_stale_removals.rs"] mod stale_removals; #[path = "tests/test_types.rs"] diff --git a/crates/sqlload/src/adapters/databricks/tests/helpers.rs b/crates/sqlload/src/adapters/databricks/tests/helpers.rs index b37037e68..23c220eb1 100644 --- a/crates/sqlload/src/adapters/databricks/tests/helpers.rs +++ b/crates/sqlload/src/adapters/databricks/tests/helpers.rs @@ -54,3 +54,77 @@ pub(crate) fn listed_load( folder, } } + +/// The `COPY INTO` of two Parquet files under `s3://lake/orders/` (one in the partition folder `day=1/`, holding `2026-11-17`; one, whose name holds a quote and a `+`, with no partition value), batch 2 of load L1: into `main.raw.orders` itself, or into `main.raw.orders_stage` when `staged`. The files hold `id` as `ID`; straight into the table a column `gone` no file holds is written, into the staging table a JSON column `payload` the files hold as `Payload`. +pub(crate) fn s3_copy_sql(staged: bool) -> String { + use crate::adapters::contract::models::WriteTarget; + use crate::adapters::contract::types::SqlAdapter; + use crate::adapters::databricks::_helpers::write::s3_copy::{ + S3Batch, S3File, s3_copy_statement, + }; + use crate::adapters::databricks::models::DatabricksSql; + use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_ROW_COLUMN}; + use crate::load_naming::models::Compat; + + let column = |name: &str, column_type: ColumnType| DesiredColumn { + name: name.to_owned(), + column_type, + origin: ColumnOrigin::Source, + }; + let last = [ + column("gone", ColumnType::Text), + column("payload", ColumnType::Json), + ]; + let columns = vec![ + column("id", ColumnType::Int), + column("day", ColumnType::Date), + column(SOURCE_FILE_COLUMN, ColumnType::Text), + column(SOURCE_ROW_COLUMN, ColumnType::Int), + last[usize::from(staged)].clone(), + column("_sqlload_load_id", ColumnType::Text), + column("_sqlload_loaded_at", ColumnType::TimestampTz), + ]; + let staging = TableName { + table: "orders_stage".to_owned(), + ..table() + }; + let target = WriteTarget { + table: &table(), + staging: Some(&staging).filter(|_| staged), + columns: &columns, + write_columns: &columns, + compat: Compat::Sqlload, + load_id: "L1", + batch: 2, + direct: !staged, + builds: false, + constants: &[], + }; + let file = |key: &str, day: Option<&str>| S3File { + key: key.to_owned(), + constants: vec![ + ("day".to_owned(), day.map(str::to_owned)), + ( + SOURCE_FILE_COLUMN.to_owned(), + Some(format!("s3://lake/orders/{key}")), + ), + ], + }; + let into = DatabricksSql.render_qualified_name(target.staging.unwrap_or(target.table)); + s3_copy_statement( + (&into, &columns), + &target, + &S3Batch { + folder: "s3://lake/orders/".to_owned(), + files: vec![ + file("day=1/a b.parquet", Some("2026-11-17")), + file("o'brien+x.parquet", None), + ], + in_files: vec![ + ("id".to_owned(), "ID".to_owned()), + ("payload".to_owned(), "Payload".to_owned()), + ], + }, + "2026-11-18T00:00:00Z", + ) +} diff --git a/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs new file mode 100644 index 000000000..218ab4b20 --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs @@ -0,0 +1,26 @@ +use crate::adapters::databricks::tests::helpers::s3_copy_sql; +use crate::adapters::databricks::tests::test_types::RenderedTestCase; + +#[test] +fn given_a_batch_of_parquet_files_in_s3_when_databricks_copies_them_then_each_row_gets_its_files_values_and_the_files_own_columns() + { + let test_cases = [ + RenderedTestCase { + description: "straight into the table: final types, the renamed column read by its name in the files, a column no file holds as NULL, per-file values picked by the decoded path", + rendered: s3_copy_sql(false), + expected_statement: "COPY INTO `main`.`raw`.`orders` FROM (SELECT CAST(`ID` AS BIGINT) AS `id`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN '2026-11-17' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN NULL ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS DATE) AS `day`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN 's3://lake/orders/day=1/a b.parquet' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN 's3://lake/orders/o\\'brien+x.parquet' ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS STRING) AS `_sqlload_source_file`, CAST(_metadata.row_index + 1 AS BIGINT) AS `_sqlload_source_row`, CAST(NULL AS STRING) AS `gone`, CAST('L1' AS STRING) AS `_sqlload_load_id`, CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` FROM 's3://lake/orders/') FILEFORMAT = PARQUET FILES = ('day=1/a b.parquet', 'o\\'brien+x.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + RenderedTestCase { + description: "into the staging table: JSON stays text there, parsed only at publish", + rendered: s3_copy_sql(true), + expected_statement: "COPY INTO `main`.`raw`.`orders_stage` FROM (SELECT CAST(`ID` AS BIGINT) AS `id`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN '2026-11-17' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN NULL ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS DATE) AS `day`, CAST(CASE url_decode(replace(_metadata.file_path, '+', '%2B')) WHEN 's3://lake/orders/day=1/a b.parquet' THEN 's3://lake/orders/day=1/a b.parquet' WHEN 's3://lake/orders/o\\'brien+x.parquet' THEN 's3://lake/orders/o\\'brien+x.parquet' ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END AS STRING) AS `_sqlload_source_file`, CAST(_metadata.row_index + 1 AS BIGINT) AS `_sqlload_source_row`, CAST(`Payload` AS STRING) AS `payload`, CAST('L1' AS STRING) AS `_sqlload_load_id`, CAST('2026-11-18T00:00:00Z' AS TIMESTAMP) AS `_sqlload_loaded_at` FROM 's3://lake/orders/') FILEFORMAT = PARQUET FILES = ('day=1/a b.parquet', 'o\\'brien+x.parquet') FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')", + }, + ]; + for test_case in &test_cases { + assert_eq!( + test_case.rendered, test_case.expected_statement, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 2e0eceee0..57e1f4a4e 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -893,15 +893,9 @@ async fn plan_table_once( schema .warnings .extend(truncated_nanoseconds(spec, live.as_deref())); - let in_s3 = spec.files.as_ref().map(|files| &files.location.store); - if let (None, Some(StoreKind::S3 { .. })) = (&schema.blocked, in_s3) - && let Err(problem) = list_reach(destination).await - { - schema = blocked_plan(vec![problem]); - } let route = match schema.blocked { Some(_) => None, - None => match table_route(spec, files.as_ref(), stage.as_ref(), destination) { + None => match table_route(spec, files.as_ref(), stage.as_ref(), destination).await { Ok(route) => Some(route), Err(problem) => { schema = blocked_plan(vec![problem]); @@ -1206,13 +1200,16 @@ fn copy_warning(route: Option<&Route>) -> Option { )) } -/// The route a table's data takes into `destination`; `read` is a files table's planned read. -pub(crate) fn table_route( +/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 it first looks up where the destination reads files itself (Databricks' external locations). +pub(crate) async fn table_route( spec: &TableSpec, read: Option<&FilesRead>, stage: Option<&StageRoute>, destination: &Destination, ) -> Result { + if let Some(StoreKind::S3 { .. }) = spec.files.as_ref().map(|files| &files.location.store) { + list_reach(destination).await?; + } let reader = reader_offer(spec, read, stage); plan_route( &reader, diff --git a/crates/sqlload/src/routing/tests/helpers.rs b/crates/sqlload/src/routing/tests/helpers.rs index c423ba1e5..589da0f5d 100644 --- a/crates/sqlload/src/routing/tests/helpers.rs +++ b/crates/sqlload/src/routing/tests/helpers.rs @@ -60,6 +60,7 @@ pub(crate) fn warehouse_destination(staging: Vec) -> WriterAccepts { rows_into: None, reads_files_itself: Some("Snowflake"), copies_files_to: Some(user_stage()), + must_pull: None, } } @@ -73,6 +74,22 @@ pub(crate) fn clickhouse_destination() -> WriterAccepts { rows_into: None, reads_files_itself: None, copies_files_to: None, + must_pull: None, + } +} + +/// Databricks: Parquet files in its volume, which SQLLoad stages, or in S3 under its one external location (the lake), which it reads in place; Parquet in S3 elsewhere it refuses. +pub(crate) fn databricks_destination() -> WriterAccepts { + let volume = Store::InternalStage("/Volumes/main/raw/stage".to_owned()); + WriterAccepts { + pulls_from: vec![volume.clone(), object_store(LAKE)], + pull_formats: vec![FileFormat::Parquet], + staging: vec![volume], + must_pull: Some(( + object_store("s3://"), + "cannot be read by Databricks: no Unity Catalog external location covers it".to_owned(), + )), + ..WriterAccepts::default() } } @@ -96,6 +113,7 @@ pub(crate) fn local_destination() -> WriterAccepts { rows_into: None, reads_files_itself: None, copies_files_to: None, + must_pull: None, } } diff --git a/crates/sqlload/src/routing/tests/test_plan_route.rs b/crates/sqlload/src/routing/tests/test_plan_route.rs index e477c749a..295b33ec3 100644 --- a/crates/sqlload/src/routing/tests/test_plan_route.rs +++ b/crates/sqlload/src/routing/tests/test_plan_route.rs @@ -1,8 +1,8 @@ use crate::routing::main::plan_route::plan_route; use crate::routing::models::{FileFormat, ReadVia, ReaderOffer, RouteOverrides, StagePreference}; use crate::routing::tests::helpers::{ - LAKE, clickhouse_destination, database_destination, duckdb_destination, files_source, - internal_stage, local_destination, object_store, streaming_source, user_stage, + LAKE, clickhouse_destination, database_destination, databricks_destination, duckdb_destination, + files_source, internal_stage, local_destination, object_store, streaming_source, user_stage, warehouse_destination, warehouse_source, }; use crate::routing::tests::test_types::{PlanRouteErrorTestCase, PlanRouteTestCase}; @@ -64,6 +64,43 @@ fn given_a_source_and_destination_when_planning_then_the_cheapest_route_is_chose "destination loads parquet files from s3://lake/scores/date=2026-11-04/", ], }, + PlanRouteTestCase { + description: "Parquet in S3 under Databricks' external location is copied where it lies", + reader: files_source("s3://lake/scores/", FileFormat::Parquet), + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_steps: &[ + "use parquet files in s3://lake/scores/", + "destination loads parquet files from s3://lake/scores/", + ], + }, + PlanRouteTestCase { + description: "CSV in S3 under Databricks' external location goes through SQLLoad into the volume", + reader: files_source("s3://lake/scores/", FileFormat::Csv), + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_steps: &[ + "use csv files in s3://lake/scores/", + "read csv into Arrow", + "write Parquet to internal stage /Volumes/main/raw/stage", + "destination loads parquet files from internal stage /Volumes/main/raw/stage", + ], + }, + PlanRouteTestCase { + description: "Parquet in S3 Databricks can't load exactly goes through SQLLoad, covered or not", + reader: ReaderOffer { + direct_blocked: Some("at is a time".to_owned()), + ..files_source("s3://other/scores/", FileFormat::Parquet) + }, + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_steps: &[ + "use parquet files in s3://other/scores/", + "read parquet into Arrow", + "write Parquet to internal stage /Volumes/main/raw/stage", + "destination loads parquet files from internal stage /Volumes/main/raw/stage", + ], + }, PlanRouteTestCase { description: "existing files the warehouse cannot reach are read and restaged only for hmac", reader: ReaderOffer { @@ -305,6 +342,13 @@ fn given_an_impossible_combination_when_planning_then_the_error_says_why() { overrides: RouteOverrides::default(), expected_message: "the destination cannot load these files exactly by reading them itself: half is Float16 in the files", }, + PlanRouteErrorTestCase { + description: "Parquet in S3 outside every external location fails Databricks, never going through SQLLoad", + reader: files_source("s3://other/events/", FileFormat::Parquet), + writer: databricks_destination(), + overrides: RouteOverrides::default(), + expected_message: "s3://other/events/ cannot be read by Databricks: no Unity Catalog external location covers it", + }, PlanRouteErrorTestCase { description: "a source with nothing to offer", reader: crate::routing::models::ReaderOffer::default(), diff --git a/website/public/index.html b/website/public/index.html index eded67b22..0849cf2c7 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -170,7 +170,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage. Parquet loads into DuckDB through DuckDB's own reader. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 3fc7dee3c..92eab842a 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -29,7 +29,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 |-------------|------------------|------------------|
 | Snowflake | `append`, `merge`, `delete_insert`, `replace` | Parquet staged on an external, user, table or named stage, then `COPY`; or Snowpipe Streaming, or `INSERT`, by `load_method`. Files under an external stage load with `COPY` directly. |
 | BigQuery (beta) | `append`, `merge`, `delete_insert`, `replace` | Arrow streamed through the Storage Write API into a staging table, then one query job per write. See [BigQuery](/docs/targets/bigquery/). |
-| Databricks (beta) | `append`, `merge`, `delete_insert`, `replace` | Parquet staged in a Unity Catalog volume, then `COPY INTO`; one statement per write. Files of a files source pass through SQLLoad. |
+| Databricks (beta) | `append`, `merge`, `delete_insert`, `replace` | Parquet in S3 copied where it lies through a Unity Catalog external location; everything else staged as Parquet in a volume, then `COPY INTO`; one statement per write. |
 | ClickHouse, one self-hosted server | `append`, `merge`, `delete_insert`, `replace` | Arrow over HTTP into a staging table, published by swapping in a whole table or whole partitions; Parquet in S3 read by ClickHouse with `s3()`. No clusters, replicated or distributed tables; ClickHouse Cloud is untested. See [ClickHouse](/docs/targets/clickhouse/). |
 | Postgres | `append`, `merge`, `delete_insert`, `replace` | Binary `COPY` into a temporary staging table, then one transaction per commit. See [Postgres](/docs/targets/postgres/). |
 | DuckDB | `append`, `merge`, `delete_insert`, `replace` | Arrow appends; Parquet files read by DuckDB with `read_parquet`. |
diff --git a/website/src/content/docs/docs/targets/databricks.mdx b/website/src/content/docs/docs/targets/databricks.mdx
index 6e922b145..88f3a1215 100644
--- a/website/src/content/docs/docs/targets/databricks.mdx
+++ b/website/src/content/docs/docs/targets/databricks.mdx
@@ -1,6 +1,6 @@
 ---
 title: "Databricks"
-description: "Load into Databricks (beta), a SQL warehouse with Unity Catalog: Parquet staged in a volume, then COPY INTO, one statement per write."
+description: "Load into Databricks (beta), a SQL warehouse with Unity Catalog: Parquet in S3 copied where it lies through an external location, everything else staged in a volume, then COPY INTO, one statement per write."
 sidebar:
   badge:
     text: Beta
@@ -9,9 +9,9 @@ sidebar:
 
 This target is in beta: it is tested, but gets less everyday use than the other connectors, so expect rough edges, and please report them in [GitHub issues](https://github.com/sqlstack-io/sqlload/issues).
 
-A Databricks target loads into a Unity Catalog SQL warehouse. SQLLoad writes each batch as Parquet
-to a Unity Catalog volume and loads it with `COPY INTO`. That is the only route: there is no
-`INSERT` fallback.
+A Databricks target loads into a Unity Catalog SQL warehouse with `COPY INTO`: from Parquet files in
+S3 where they lie, through an external location, and from Parquet SQLLoad writes to a Unity Catalog
+volume for everything else. There is no `INSERT` fallback.
 
 ## Connection
 
@@ -52,8 +52,34 @@ Each batch is written as Parquet to `/sqlload///` and lo
 next load removes folders a failed run left once they are over a day old; a folder a running load
 is still filling is kept.
 
-Files of a files source pass through SQLLoad on the way to the volume. `copy_to_internal_stage` is
-for Snowflake and is refused here.
+### Parquet files in S3
+
+Parquet files of a files source in S3 load straight from the bucket, with no row passing through
+SQLLoad, when a Unity Catalog external location covers their folder. `plan` lists the external
+locations the service principal can see, which needs `READ FILES` on the one it reads through:
+
+```sql
+CREATE EXTERNAL LOCATION lake URL 's3://example-bucket/events/' WITH (STORAGE CREDENTIAL lake_reader);
+GRANT READ FILES ON EXTERNAL LOCATION lake TO ``;
+```
+
+```text
+route   Databricks COPYs directly from s3://example-bucket/events/ (nothing through the loader)
+```
+
+Each batch is one `COPY INTO` over its files, so a batch lands whole or not at all. The columns
+SQLLoad adds (`_sqlload_source_file`, `_sqlload_source_version`, `_sqlload_source_row`, the load id
+and time) and partition values from the path are filled in the same statement, and the batch fails
+when Databricks reports inserting a different number of rows than the files' footers hold.
+
+Parquet in S3 that no external location covers fails `plan`, naming the folder and the fix; such
+files never go through SQLLoad instead. Files go through SQLLoad, staged in the volume as below,
+when Databricks can't load them exactly by reading them itself, and `plan` says why, such as for
+CSV and JSON files, a table that masks columns, has checks or flattens objects, a column held in a way a cast
+wouldn't keep exactly (nanoseconds, a type other than the one it loads as), a time, a UUID or a
+list, and files on an S3-compatible endpoint, which no external location reaches. Files in a local
+folder or over SFTP always go through SQLLoad. `copy_to_internal_stage` is for Snowflake and is
+refused here.
 
 ## Write strategies
 
@@ -117,6 +143,10 @@ values checked against each source after every round.
 | BigQuery | not run | live | not run | live |
 | Files on an SFTP server (gzip CSV, Parquet) | live | not run | not run | refused at `plan` |
 
+Parquet in S3 copied through an external location was also run live with `append`, `replace` and
+`delete_insert` (a staging table), each table compared row for row, lineage columns included, with
+the same files loaded through SQLLoad.
+
 `merge` is refused for files sources on every target. The live rounds also covered incremental
 loads on a cursor, rows with a NULL cursor under `delete_insert`, an added column, widening a
 decimal in place, `contract diff`, `contract diff --source` and `contract generate --write`, and a
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index c084f7251..56340a2ee 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -15,7 +15,7 @@ names the route for every table.
 | DuckDB | DuckDB reads them with `read_parquet` | Read by SQLLoad, appended as Arrow | Appended as Arrow |
 | MotherDuck (beta) | Read by SQLLoad, appended as Arrow | Read by SQLLoad, appended as Arrow | Appended as Arrow |
 | Snowflake | `COPY` from an external stage over the bucket | `COPY` from an external stage over the bucket | Written as Parquet to the `load_method`'s stage, then `COPY` (or streamed or inserted, if the `load_method` says so) |
-| Databricks (beta) | Staged by SQLLoad in the volume, then `COPY INTO` | Read by SQLLoad, staged as Parquet in the volume, then `COPY INTO` | Written as Parquet to the volume, then `COPY INTO` |
+| Databricks (beta) | In S3: `COPY INTO` where they lie, through an external location. In a folder: staged by SQLLoad in the volume, then `COPY INTO` | Read by SQLLoad, staged as Parquet in the volume, then `COPY INTO` | Written as Parquet to the volume, then `COPY INTO` |
 | BigQuery (beta) | Read by SQLLoad, streamed as Arrow into a staging table, then one query job | As for Parquet files | Streamed as Arrow into a staging table, then one query job |
 | ClickHouse | In S3: ClickHouse reads them with `s3()` into a staging table. In a folder: read by SQLLoad, sent as Arrow | Read by SQLLoad, sent as Arrow into a staging table | Sent as Arrow into a staging table, then swapped in by table or partition |
 | Postgres | Read by SQLLoad, then binary `COPY` into a temporary staging table | As for Parquet files | Binary `COPY` into a temporary staging table, then one transaction per commit |

From fadfa150921d2ee5a0657471b53b9ff91b11810c Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Wed, 18 Nov 2026 19:44:54 +0000
Subject: [PATCH 3/4] wip: Databricks external locations: check READ FILES at
 plan, list only for in-place tables, s3a/s3n URLs, split COPYs by FILES limit
 and Parquet types, take back a failed direct append, endpoints from the
 environment

---
 .../contract/main/_custom_endpoint.rs         |  22 +++
 .../sqlload/src/adapters/contract/main/mod.rs |   1 +
 .../databricks/_helpers/stage/locations.rs    |  13 ++
 .../adapters/databricks/_helpers/stage/mod.rs |   1 +
 .../databricks/_helpers/write/direct_load.rs  | 133 ++++++++++++++----
 .../databricks/_helpers/write/s3_copy.rs      |  35 +++++
 .../src/adapters/databricks/constants.rs      |   3 +
 .../databricks/main/stage/_check_folder.rs    |  20 +++
 .../databricks/main/stage/_direct_problem.rs  |  15 +-
 .../main/stage/_external_locations.rs         |   3 +-
 .../src/adapters/databricks/main/stage/mod.rs |   1 +
 .../adapters/databricks/tests/test_s3_copy.rs | 104 +++++++++++++-
 .../adapters/databricks/tests/test_types.rs   |  14 ++
 .../dispatch/main/route/list_reach.rs         |  27 +++-
 .../adapters/duckdb/_helpers/direct_checks.rs |  20 ---
 .../adapters/duckdb/main/_direct_problem.rs   |   3 +-
 .../adapters/duckdb/tests/test_s3_secret.rs   |   2 +-
 .../sqlload/src/loader/_helpers/table_plan.rs |  13 +-
 .../src/routing/_helpers/files_route.rs       |   6 +
 .../src/routing/tests/test_plan_route.rs      |  31 +++-
 .../sqlload/src/routing/tests/test_types.rs   |   8 ++
 .../content/docs/docs/targets/databricks.mdx  |  25 ++--
 22 files changed, 426 insertions(+), 74 deletions(-)
 create mode 100644 crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs
 create mode 100644 crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs
 create mode 100644 crates/sqlload/src/adapters/databricks/main/stage/_check_folder.rs

diff --git a/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs b/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs
new file mode 100644
index 000000000..667f3b6b3
--- /dev/null
+++ b/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs
@@ -0,0 +1,22 @@
+//! Whether an S3 store points at a custom endpoint, which a warehouse reading S3 itself would not follow.
+
+use crate::store_paths::models::StoreKind;
+
+/// Why an S3 store points somewhere a warehouse reading S3 itself would not read: an endpoint in the connection or in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows.
+pub(in crate::adapters) fn custom_endpoint(
+    store: &StoreKind,
+    variable: &dyn Fn(&str) -> Option,
+) -> Option {
+    let from_env = ["AWS_ENDPOINT", "AWS_ENDPOINT_URL"]
+        .iter()
+        .any(|name| variable(name).is_some_and(|value| !value.is_empty()));
+    match store {
+        StoreKind::S3 {
+            endpoint: Some(_), ..
+        } => Some("the S3 connection sets a custom endpoint".to_owned()),
+        StoreKind::S3 { .. } if from_env => {
+            Some("AWS_ENDPOINT or AWS_ENDPOINT_URL sets a custom S3 endpoint".to_owned())
+        }
+        _ => None,
+    }
+}
diff --git a/crates/sqlload/src/adapters/contract/main/mod.rs b/crates/sqlload/src/adapters/contract/main/mod.rs
index 1d80cd6b2..b5fc0a7f7 100644
--- a/crates/sqlload/src/adapters/contract/main/mod.rs
+++ b/crates/sqlload/src/adapters/contract/main/mod.rs
@@ -1,4 +1,5 @@
 pub(in crate::adapters) mod _batch_direct_problem;
+pub(in crate::adapters) mod _custom_endpoint;
 pub(in crate::adapters) mod _dropped_ordinals;
 pub(in crate::adapters) mod _files_direct_problem;
 pub(in crate::adapters) mod _footer_rows;
diff --git a/crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs b/crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs
new file mode 100644
index 000000000..c9696345e
--- /dev/null
+++ b/crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs
@@ -0,0 +1,13 @@
+//! How SQLLoad compares folders with Databricks' external locations.
+
+/// An external location's `url` as SQLLoad compares folders with it: `s3://` for the `s3a://` and `s3n://` it may be stored as, ending in `/` so that it only covers what lies below it.
+pub(crate) fn location_url(url: &str) -> String {
+    let url = url.trim_end_matches('/');
+    let s3 = ["s3a://", "s3n://"]
+        .iter()
+        .find_map(|scheme| url.strip_prefix(scheme));
+    match s3 {
+        Some(rest) => format!("s3://{rest}/"),
+        None => format!("{url}/"),
+    }
+}
diff --git a/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs b/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs
index f149669a3..b858ed8da 100644
--- a/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs
+++ b/crates/sqlload/src/adapters/databricks/_helpers/stage/mod.rs
@@ -1 +1,2 @@
 pub(crate) mod folders;
+pub(crate) mod locations;
diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs
index 39a451ce7..2ba363864 100644
--- a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs
+++ b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs
@@ -2,22 +2,27 @@
 
 use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem;
 use crate::adapters::contract::main::_footer_rows::footer_rows;
-use crate::adapters::contract::models::{DirectBatch, DirectLoaded};
+use crate::adapters::contract::models::{DirectBatch, DirectLoaded, TableName};
 use crate::adapters::contract::types::SqlAdapter;
-use crate::adapters::databricks::_helpers::write::s3_copy::{S3Batch, S3File, s3_copy_statement};
-use crate::adapters::databricks::constants::INSERTED_ROWS_COLUMN;
+use crate::adapters::databricks::_helpers::write::s3_copy::{
+    S3Batch, S3File, copy_groups, s3_copy_statement,
+};
+use crate::adapters::databricks::constants::{INSERTED_ROWS_COLUMN, MAX_FILES_PER_COPY};
 use crate::adapters::databricks::main::stage::_direct_problem::direct_problem;
 use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter};
 use crate::databricks_api::main::execute_statement::execute_statement;
 use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN};
 use crate::files_source::main::normal_value::normal_value;
 use crate::files_source::models::DataFile;
+use crate::load_naming::main::metadata_name::metadata_name;
+use crate::load_naming::models::MetadataRole;
 use crate::loader::errors::LoaderError;
 use crate::schema_plan::models::DesiredColumn;
 use crate::sql_api::models::SqlRows;
+use arrow_schema::Schema;
 
 impl DatabricksWriter {
-    /// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, in one `COPY INTO` (one transaction) into the staging or build table the load publishes from, or into the table itself. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose inserted rows, as `COPY INTO` reports them, differ from its footers' (an append into the table itself has then already committed them); files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location.
+    /// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, into the staging or build table the load publishes from, or into the table itself: one `COPY INTO` (one transaction) per [`copy_groups`] group, as `FILES` takes at most 1000 files and `mergeSchema` won't merge one column held as two types. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose inserted rows, as `COPY INTO` reports them, differ from its footers'; an append into the table itself first takes back what it committed. Files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location.
     pub(in crate::adapters) async fn copy_files(
         &mut self,
         batch: &DirectBatch<'_>,
@@ -46,36 +51,40 @@ impl DatabricksWriter {
             (Some(staging), false) => (staging, target.columns),
             (staging, _) => (staging.unwrap_or(target.table), batch.write_columns),
         };
-        let mut files = Vec::with_capacity(batch.files.len());
-        for data in batch.files {
-            files.push(s3_file(batch, data, columns)?);
-        }
-        let mut in_files: Vec<(String, String)> = Vec::new();
-        for footer in batch.footers.parquet.values() {
-            for field in footer.schema().fields() {
-                let loaded = batch.spec.source_table.loaded_as(field.name()).to_owned();
-                if !in_files.iter().any(|(name, _)| *name == loaded) {
-                    in_files.push((loaded, field.name().clone()));
-                }
+        let no_footer = Schema::empty();
+        let schemas: Vec<(&DataFile, &Schema)> = batch
+            .files
+            .iter()
+            .map(|data| {
+                let schema = batch
+                    .footers
+                    .parquet
+                    .get(&data.file.store_key())
+                    .map_or(&no_footer, |footer| footer.schema().as_ref());
+                (data, schema)
+            })
+            .collect();
+        let mut inserted = 0;
+        let mut copied = false;
+        for group in copy_groups(schemas, MAX_FILES_PER_COPY) {
+            let statement = group_statement(batch, &group, (into, columns), folder.clone())?;
+            let result = match execute_statement(&self.destination.api, &statement).await {
+                Ok(result) => inserted_rows(&result),
+                Err(error) => Err(error.into()),
+            };
+            match result {
+                Ok(rows) => inserted += rows,
+                Err(error) if copied => return Err(self.undo(batch, error).await),
+                Err(error) => return Err(error),
             }
+            copied = true;
         }
-        let statement = s3_copy_statement(
-            (&DatabricksSql.render_qualified_name(into), columns),
-            target,
-            &S3Batch {
-                folder,
-                files,
-                in_files,
-            },
-            batch.loaded_at,
-        );
-        let result = execute_statement(&self.destination.api, &statement).await?;
-        let inserted = inserted_rows(&result)?;
         let expected = footer_rows(batch.footers);
         if inserted != expected {
-            return Err(LoaderError::Invalid(format!(
+            let error = LoaderError::Invalid(format!(
                 "Databricks loaded {inserted} rows from this batch's files, but their footers hold {expected}"
-            )));
+            ));
+            return Err(self.undo(batch, error).await);
         }
         Ok(Some(match target.staging {
             Some(_) => DirectLoaded::Staged(expected),
@@ -84,6 +93,72 @@ impl DatabricksWriter {
     }
 }
 
+impl DatabricksWriter {
+    /// After `error`, take back the rows a batch going straight into the table committed: its load's rows from its files, which a later run would otherwise skip as loaded. A batch landing in a staging table needs nothing, as the load drops it.
+    async fn undo(&self, batch: &DirectBatch<'_>, error: LoaderError) -> LoaderError {
+        let target = &batch.target;
+        if target.staging.is_some() {
+            return error;
+        }
+        let Some(source) = batch.spec.files.as_ref() else {
+            return error;
+        };
+        let literal = |text: &str| DatabricksSql.render_loader_value_literal(text);
+        let paths: Vec = batch
+            .files
+            .iter()
+            .map(|data| literal(&format!("{}{}", source.location, data.file.relative_key)))
+            .collect();
+        let statement = format!(
+            "DELETE FROM {} WHERE {} = {} AND {} IN ({})",
+            DatabricksSql.render_qualified_name(target.table),
+            DatabricksSql.render_identifier(metadata_name(target.compat, MetadataRole::LoadId)),
+            literal(target.load_id),
+            DatabricksSql.render_identifier(SOURCE_FILE_COLUMN),
+            paths.join(", ")
+        );
+        match execute_statement(&self.destination.api, &statement).await {
+            Ok(_) => error,
+            Err(undo) => LoaderError::Invalid(format!(
+                "{error}; taking back the rows it committed then failed too, so the table holds them: {undo}"
+            )),
+        }
+    }
+}
+
+/// The `COPY INTO` that loads `group`, files of `batch` below `folder`, into `into`'s `columns`, reading each table column the group's files hold by its name in them.
+fn group_statement(
+    batch: &DirectBatch<'_>,
+    group: &[&DataFile],
+    (into, columns): (&TableName, &[DesiredColumn]),
+    folder: String,
+) -> Result {
+    let mut files = Vec::with_capacity(group.len());
+    let mut in_files: Vec<(String, String)> = Vec::new();
+    for data in group {
+        files.push(s3_file(batch, data, columns)?);
+        let Some(footer) = batch.footers.parquet.get(&data.file.store_key()) else {
+            continue;
+        };
+        for field in footer.schema().fields() {
+            let loaded = batch.spec.source_table.loaded_as(field.name()).to_owned();
+            if !in_files.iter().any(|(name, _)| *name == loaded) {
+                in_files.push((loaded, field.name().clone()));
+            }
+        }
+    }
+    Ok(s3_copy_statement(
+        (&DatabricksSql.render_qualified_name(into), columns),
+        &batch.target,
+        &S3Batch {
+            folder,
+            files,
+            in_files,
+        },
+        batch.loaded_at,
+    ))
+}
+
 /// The rows a `COPY INTO`'s `result` reports it inserted.
 fn inserted_rows(result: &SqlRows) -> Result {
     let reported = result
diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs
index 71b77b37b..606f00984 100644
--- a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs
+++ b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs
@@ -9,6 +9,7 @@ use crate::files_source::constants::SOURCE_ROW_COLUMN;
 use crate::load_naming::main::metadata_role::metadata_role;
 use crate::load_naming::models::MetadataRole;
 use crate::schema_plan::models::DesiredColumn;
+use arrow_schema::{Field, Schema};
 
 /// The path `COPY INTO` reads each row from, as SQLLoad names it: `_metadata.file_path` is percent-encoded, except for `+`, which `url_decode` alone would read as a space.
 const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))";
@@ -101,3 +102,37 @@ fn per_file(batch: &S3Batch, given: &[(&S3File, Option<&String>)]) -> String {
         picks.join(" ")
     )
 }
+
+/// `files`, in order, in the `COPY INTO`s that load them: at most `limit` to each (the most `FILES` takes), and never two that hold one column as different types, which `mergeSchema` refuses to merge (an INT32 in one file and an INT64 in another).
+pub(crate) fn copy_groups(files: Vec<(T, &Schema)>, limit: usize) -> Vec> {
+    let mut groups: Vec<(Vec, Vec<&Field>)> = Vec::new();
+    for (file, schema) in files {
+        let joins = groups
+            .iter()
+            .position(|(members, held)| members.len() < limit && agrees(held, schema));
+        let index = joins.unwrap_or_else(|| {
+            groups.push((Vec::new(), Vec::new()));
+            groups.len() - 1
+        });
+        let (members, held) = &mut groups[index];
+        members.push(file);
+        for field in schema.fields() {
+            if !held.iter().any(|other| other.name() == field.name()) {
+                held.push(field);
+            }
+        }
+    }
+    groups.into_iter().map(|(members, _)| members).collect()
+}
+
+/// Whether `schema` holds every column it shares with `held` as the same type.
+fn agrees(held: &[&Field], schema: &Schema) -> bool {
+    for field in schema.fields() {
+        for other in held {
+            if other.name() == field.name() && other.data_type() != field.data_type() {
+                return false;
+            }
+        }
+    }
+    true
+}
diff --git a/crates/sqlload/src/adapters/databricks/constants.rs b/crates/sqlload/src/adapters/databricks/constants.rs
index c71d8ee47..62579e920 100644
--- a/crates/sqlload/src/adapters/databricks/constants.rs
+++ b/crates/sqlload/src/adapters/databricks/constants.rs
@@ -26,6 +26,9 @@ pub const COPY_OPTIONS: &str = "FILEFORMAT = PARQUET COPY_OPTIONS ('force' = 'tr
 pub const S3_COPY_OPTIONS: &str =
     "FORMAT_OPTIONS ('mergeSchema' = 'true') COPY_OPTIONS ('force' = 'true')";
 
+/// The most files one `COPY INTO`'s `FILES` list takes.
+pub const MAX_FILES_PER_COPY: usize = 1000;
+
 /// The external locations the service principal can see.
 pub const LIST_LOCATIONS: &str = "SHOW EXTERNAL LOCATIONS";
 
diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_check_folder.rs b/crates/sqlload/src/adapters/databricks/main/stage/_check_folder.rs
new file mode 100644
index 000000000..08a25b38f
--- /dev/null
+++ b/crates/sqlload/src/adapters/databricks/main/stage/_check_folder.rs
@@ -0,0 +1,20 @@
+//! Check that Databricks can read a folder of files through its external location.
+
+use crate::adapters::contract::types::SqlAdapter;
+use crate::adapters::databricks::models::{DatabricksDestination, DatabricksSql};
+use crate::databricks_api::main::execute_statement::execute_statement;
+use crate::http::errors::ApiError;
+
+/// Check that `destination`'s service principal can read `folder`, which a location it sees covers: seeing a location does not grant READ FILES on it, which `LIST` needs as `COPY INTO` does.
+pub(in crate::adapters) async fn check_folder(
+    destination: &DatabricksDestination,
+    folder: &str,
+) -> Result<(), ApiError> {
+    let statement = format!(
+        "LIST {} LIMIT 1",
+        DatabricksSql.render_loader_value_literal(folder)
+    );
+    execute_statement(&destination.api, &statement)
+        .await
+        .map(|_| ())
+}
diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs b/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs
index 2d2db6924..a27ba6f07 100644
--- a/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs
+++ b/crates/sqlload/src/adapters/databricks/main/stage/_direct_problem.rs
@@ -1,5 +1,6 @@
 //! Whether Databricks may load a files table's files itself.
 
+use crate::adapters::contract::main::_custom_endpoint::custom_endpoint;
 use crate::adapters::contract::main::_files_direct_problem::files_direct_problem;
 use crate::column_types::models::ColumnType;
 use crate::files_source::models::{FilesRead, SourceFormat};
@@ -9,14 +10,14 @@ use crate::store_paths::models::StoreKind;
 /// Why Databricks cannot load the planned `read` of a files table in S3 exactly by `COPY`ing its Parquet files where they lie, so SQLLoad reads them into the staging volume instead; `None` when it can, or when the files aren't in S3. Every column must load as the type the files hold it as, so Databricks' casts never differ from SQLLoad's; an external location reaches only AWS S3, not an S3-compatible endpoint; masks need SQLLoad's reader; and Databricks reads a Parquet time as text in its own format, a UUID as bytes, and lists as its own arrays.
 pub(in crate::adapters) fn direct_problem(spec: &TableSpec, read: &FilesRead) -> Option {
     let files = spec.files.as_ref()?;
-    let StoreKind::S3 { endpoint, .. } = &files.location.store else {
+    let StoreKind::S3 { .. } = &files.location.store else {
         return None;
     };
     let problems = [
         (files.encoding.format != SourceFormat::Parquet)
             .then(|| "only Parquet files are read by Databricks itself".to_owned()),
-        endpoint.as_ref().map(|endpoint| {
-            format!("the files are on {endpoint}, which no Unity Catalog external location reaches")
+        custom_endpoint(&files.location.store, &environment_variable).map(|endpoint| {
+            format!("{endpoint}, which no Unity Catalog external location reaches")
         }),
         (!spec.masked.columns.is_empty()).then(|| "the table masks columns".to_owned()),
         files_direct_problem(files, read),
@@ -35,3 +36,11 @@ pub(in crate::adapters) fn direct_problem(spec: &TableSpec, read: &FilesRead) ->
     ];
     problems.into_iter().flatten().next()
 }
+
+/// `name`'s value in the process environment; unset when it isn't Unicode, as SQLLoad's reader takes it.
+fn environment_variable(name: &str) -> Option {
+    match std::env::var(name) {
+        Ok(value) => Some(value),
+        Err(std::env::VarError::NotPresent | std::env::VarError::NotUnicode(_)) => None,
+    }
+}
diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs
index ebe837c88..f39a66d1e 100644
--- a/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs
+++ b/crates/sqlload/src/adapters/databricks/main/stage/_external_locations.rs
@@ -1,5 +1,6 @@
 //! The Unity Catalog external locations a Databricks destination reads files through.
 
+use crate::adapters::databricks::_helpers::stage::locations::location_url;
 use crate::adapters::databricks::constants::{LIST_LOCATIONS, LOCATION_URL_COLUMN};
 use crate::adapters::databricks::models::DatabricksDestination;
 use crate::databricks_api::main::execute_statement::execute_statement;
@@ -24,7 +25,7 @@ pub(in crate::adapters) async fn external_locations(
                 rows.rows
                     .iter()
                     .filter_map(|row| row.get(url).cloned().flatten())
-                    .map(|url| format!("{}/", url.trim_end_matches('/')))
+                    .map(|url| location_url(&url))
                     .collect(),
             )
         })
diff --git a/crates/sqlload/src/adapters/databricks/main/stage/mod.rs b/crates/sqlload/src/adapters/databricks/main/stage/mod.rs
index 51e3be694..55a6d787d 100644
--- a/crates/sqlload/src/adapters/databricks/main/stage/mod.rs
+++ b/crates/sqlload/src/adapters/databricks/main/stage/mod.rs
@@ -1,3 +1,4 @@
+pub(in crate::adapters) mod _check_folder;
 pub(in crate::adapters) mod _check_volume;
 pub(in crate::adapters) mod _direct_problem;
 pub(in crate::adapters) mod _external_locations;
diff --git a/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs
index 218ab4b20..f6552d3b8 100644
--- a/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs
+++ b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs
@@ -1,5 +1,10 @@
+use crate::adapters::databricks::_helpers::stage::locations::location_url;
+use crate::adapters::databricks::_helpers::write::s3_copy::copy_groups;
 use crate::adapters::databricks::tests::helpers::s3_copy_sql;
-use crate::adapters::databricks::tests::test_types::RenderedTestCase;
+use crate::adapters::databricks::tests::test_types::{
+    CopyGroupsTestCase, LocationUrlTestCase, RenderedTestCase,
+};
+use arrow_schema::{DataType, Field, Schema};
 
 #[test]
 fn given_a_batch_of_parquet_files_in_s3_when_databricks_copies_them_then_each_row_gets_its_files_values_and_the_files_own_columns()
@@ -24,3 +29,100 @@ fn given_a_batch_of_parquet_files_in_s3_when_databricks_copies_them_then_each_ro
         );
     }
 }
+
+#[test]
+fn given_a_batchs_parquet_files_when_databricks_copies_them_then_each_copy_holds_files_whose_shared_columns_agree_and_no_more_than_files_takes()
+ {
+    let test_cases = [
+        CopyGroupsTestCase {
+            description: "files that agree, or hold different columns, share one copy",
+            files: vec![
+                ("a", vec![("id", DataType::Int64)]),
+                ("b", vec![("id", DataType::Int64), ("name", DataType::Utf8)]),
+                ("c", vec![("name", DataType::Utf8)]),
+            ],
+            limit: 1000,
+            expected_groups: vec![vec!["a", "b", "c"]],
+        },
+        CopyGroupsTestCase {
+            description: "an INT32 and an INT64 of one column go to separate copies, and a later file joins the first copy it agrees with",
+            files: vec![
+                ("a", vec![("id", DataType::Int32)]),
+                ("b", vec![("id", DataType::Int64)]),
+                ("c", vec![("id", DataType::Int32), ("x", DataType::Utf8)]),
+                ("d", vec![("x", DataType::Int64)]),
+            ],
+            limit: 1000,
+            expected_groups: vec![vec!["a", "c"], vec!["b", "d"]],
+        },
+        CopyGroupsTestCase {
+            description: "no copy holds more files than the limit",
+            files: vec![
+                ("a", vec![("id", DataType::Int64)]),
+                ("b", vec![("id", DataType::Int64)]),
+                ("c", vec![("id", DataType::Int64)]),
+            ],
+            limit: 2,
+            expected_groups: vec![vec!["a", "b"], vec!["c"]],
+        },
+    ];
+    for test_case in test_cases {
+        let schemas: Vec<(&str, Schema)> = test_case
+            .files
+            .iter()
+            .map(|(name, columns)| {
+                let fields: Vec = columns
+                    .iter()
+                    .map(|(column, data_type)| Field::new(*column, data_type.clone(), true))
+                    .collect();
+                (*name, Schema::new(fields))
+            })
+            .collect();
+        let groups = copy_groups(
+            schemas
+                .iter()
+                .map(|(name, schema)| (*name, schema))
+                .collect(),
+            test_case.limit,
+        );
+        assert_eq!(
+            groups, test_case.expected_groups,
+            "{}",
+            test_case.description
+        );
+    }
+}
+
+#[test]
+fn given_an_external_locations_url_when_comparing_folders_with_it_then_it_is_an_s3_folder() {
+    let test_cases = [
+        LocationUrlTestCase {
+            description: "an s3:// URL without a trailing slash",
+            url: "s3://lake/raw",
+            expected_url: "s3://lake/raw/",
+        },
+        LocationUrlTestCase {
+            description: "an s3a:// URL",
+            url: "s3a://lake/raw/",
+            expected_url: "s3://lake/raw/",
+        },
+        LocationUrlTestCase {
+            description: "an s3n:// URL of a whole bucket",
+            url: "s3n://lake",
+            expected_url: "s3://lake/",
+        },
+        LocationUrlTestCase {
+            description: "an Azure URL stays as it is",
+            url: "abfss://box@account.dfs.core.windows.net/raw",
+            expected_url: "abfss://box@account.dfs.core.windows.net/raw/",
+        },
+    ];
+    for test_case in test_cases {
+        assert_eq!(
+            location_url(test_case.url),
+            test_case.expected_url,
+            "{}",
+            test_case.description
+        );
+    }
+}
diff --git a/crates/sqlload/src/adapters/databricks/tests/test_types.rs b/crates/sqlload/src/adapters/databricks/tests/test_types.rs
index 036442524..146d4e041 100644
--- a/crates/sqlload/src/adapters/databricks/tests/test_types.rs
+++ b/crates/sqlload/src/adapters/databricks/tests/test_types.rs
@@ -20,3 +20,17 @@ pub(crate) struct RenderedTestCase {
     pub(crate) rendered: String,
     pub(crate) expected_statement: &'static str,
 }
+
+pub(crate) struct CopyGroupsTestCase {
+    pub(crate) description: &'static str,
+    /// Each file's name and its columns, by name and type.
+    pub(crate) files: Vec<(&'static str, Vec<(&'static str, arrow_schema::DataType)>)>,
+    pub(crate) limit: usize,
+    pub(crate) expected_groups: Vec>,
+}
+
+pub(crate) struct LocationUrlTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) url: &'static str,
+    pub(crate) expected_url: &'static str,
+}
diff --git a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs
index 8f69542e4..b6c4cb5c2 100644
--- a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs
+++ b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs
@@ -1,15 +1,32 @@
 //! Where a destination reads files itself, beyond its own stores.
 
+use crate::adapters::databricks::main::stage::_check_folder::check_folder;
 use crate::adapters::databricks::main::stage::_external_locations::external_locations;
 use crate::adapters::dispatch::models::Destination;
 
-/// Look up, once per run, where `destination` reads files itself beyond its own stores, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers: Databricks' Unity Catalog external locations. Called at plan for a table whose files are in S3; the error says why they can't be listed.
-pub async fn list_reach(destination: &Destination) -> Result<(), String> {
+/// Look up, once per run, where `destination` reads files itself beyond its own stores, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers: Databricks' Unity Catalog external locations; and check that it can read `folder`, which holds files to load, when one of them covers it (a folder holding no files yet does not list). Called at plan for a table whose files in S3 the destination would read where they lie; the error says why it can't, with the fix.
+pub async fn list_reach(destination: &Destination, folder: Option<&str>) -> Result<(), String> {
     let Destination::Databricks(databricks) = destination else {
         return Ok(());
     };
-    external_locations(databricks)
+    let locations = external_locations(databricks)
         .await
-        .map(|_| ())
-        .map_err(|error| format!("cannot list Databricks' external locations: {error}"))
+        .map_err(|error| format!("cannot list Databricks' external locations: {error}"))?;
+    let Some(folder) = folder else {
+        return Ok(());
+    };
+    let Some(url) = locations
+        .iter()
+        .find(|url| folder.starts_with(url.as_str()))
+    else {
+        return Ok(());
+    };
+    check_folder(databricks, folder).await.map_err(|error| {
+        let error = error.to_string();
+        let first_line = error.lines().next().unwrap_or_default();
+        format!(
+            "{folder} cannot be read by Databricks through the external location at {url}: {}. Fix: grant the service principal READ FILES on that external location",
+            first_line.trim_end_matches([':', ' '])
+        )
+    })
 }
diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs b/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs
index 2e83ebbb2..cdda522b4 100644
--- a/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs
+++ b/crates/sqlload/src/adapters/duckdb/_helpers/direct_checks.rs
@@ -1,26 +1,6 @@
 //! The DuckDB-only checks that decide whether DuckDB may read a files table's files itself.
 
 use crate::files_source::models::FileColumn;
-use crate::store_paths::models::StoreKind;
-
-/// Why an S3 store points somewhere DuckDB would not read: an endpoint in the connection or in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows.
-pub(crate) fn custom_endpoint(
-    store: &StoreKind,
-    variable: &dyn Fn(&str) -> Option,
-) -> Option {
-    let from_env = ["AWS_ENDPOINT", "AWS_ENDPOINT_URL"]
-        .iter()
-        .any(|name| variable(name).is_some_and(|value| !value.is_empty()));
-    match store {
-        StoreKind::S3 {
-            endpoint: Some(_), ..
-        } => Some("the S3 connection sets a custom endpoint".to_owned()),
-        StoreKind::S3 { .. } if from_env => {
-            Some("AWS_ENDPOINT or AWS_ENDPOINT_URL sets a custom S3 endpoint".to_owned())
-        }
-        _ => None,
-    }
-}
 
 /// A file column whose name DuckDB's file reader uses itself, in any case: such a column would take the place of `file_index`, which picks each row's per-file values.
 pub(crate) fn reserved_name(column: &FileColumn) -> Option {
diff --git a/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs b/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs
index 3c75f3491..1aaa2388e 100644
--- a/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs
+++ b/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs
@@ -1,7 +1,8 @@
 //! Whether DuckDB may load a files table's files itself.
 
+use crate::adapters::contract::main::_custom_endpoint::custom_endpoint;
 use crate::adapters::contract::main::_files_direct_problem::files_direct_problem;
-use crate::adapters::duckdb::_helpers::direct_checks::{custom_endpoint, reserved_name};
+use crate::adapters::duckdb::_helpers::direct_checks::reserved_name;
 use crate::adapters::duckdb::_helpers::motherduck_connection::environment_variable;
 use crate::files_source::models::{FilesRead, SourceFormat};
 use crate::loader::models::TableSpec;
diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs
index 3d18e2a5b..23f9bcc66 100644
--- a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs
+++ b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs
@@ -1,4 +1,4 @@
-use crate::adapters::duckdb::_helpers::direct_checks::custom_endpoint;
+use crate::adapters::contract::main::_custom_endpoint::custom_endpoint;
 use crate::adapters::duckdb::_helpers::pull_sql::{default_region, s3_secret};
 use crate::adapters::duckdb::tests::test_types::{AwsVariablesTestCase, S3SecretTestCase};
 use crate::store_paths::models::StoreKind;
diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs
index 57e1f4a4e..0a71a9b73 100644
--- a/crates/sqlload/src/loader/_helpers/table_plan.rs
+++ b/crates/sqlload/src/loader/_helpers/table_plan.rs
@@ -1200,17 +1200,22 @@ fn copy_warning(route: Option<&Route>) -> Option {
     ))
 }
 
-/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 it first looks up where the destination reads files itself (Databricks' external locations).
+/// The route a table's data takes into `destination`; `read` is a files table's planned read. For files in S3 that nothing sends through SQLLoad, it first looks up where the destination reads files itself (Databricks' external locations).
 pub(crate) async fn table_route(
     spec: &TableSpec,
     read: Option<&FilesRead>,
     stage: Option<&StageRoute>,
     destination: &Destination,
 ) -> Result {
-    if let Some(StoreKind::S3 { .. }) = spec.files.as_ref().map(|files| &files.location.store) {
-        list_reach(destination).await?;
-    }
     let reader = reader_offer(spec, read, stage);
+    let in_place = reader.needs_loader.is_none() && reader.direct_blocked.is_none();
+    if let Some(files) = spec.files.as_ref().filter(|_| in_place)
+        && let StoreKind::S3 { .. } = files.location.store
+    {
+        let folder = files.location.to_string();
+        let holds_files = read.is_some_and(|read| !read.files.is_empty());
+        list_reach(destination, holds_files.then_some(folder.as_str())).await?;
+    }
     plan_route(
         &reader,
         &writer_accepts(destination, stage),
diff --git a/crates/sqlload/src/routing/_helpers/files_route.rs b/crates/sqlload/src/routing/_helpers/files_route.rs
index 11efe60ad..f2cd52a33 100644
--- a/crates/sqlload/src/routing/_helpers/files_route.rs
+++ b/crates/sqlload/src/routing/_helpers/files_route.rs
@@ -71,6 +71,12 @@ pub(crate) fn other_reads(
             overrides,
             &format!("{LOADER_READS_BECAUSE} {reason}"),
         ),
+        (false, None, Some(reason)) if must_pull(writer, files).is_some() => loader_reads(
+            files,
+            writer,
+            overrides,
+            &format!("{LOADER_READS_BECAUSE} {reason}"),
+        ),
         (false, None, None) if must_pull(writer, files).is_some() => Err(RouteError::Uncovered {
             location: files.store.to_string(),
             reason: must_pull(writer, files).unwrap_or_default(),
diff --git a/crates/sqlload/src/routing/tests/test_plan_route.rs b/crates/sqlload/src/routing/tests/test_plan_route.rs
index 295b33ec3..2fb1f637f 100644
--- a/crates/sqlload/src/routing/tests/test_plan_route.rs
+++ b/crates/sqlload/src/routing/tests/test_plan_route.rs
@@ -5,7 +5,9 @@ use crate::routing::tests::helpers::{
     files_source, internal_stage, local_destination, object_store, streaming_source, user_stage,
     warehouse_destination, warehouse_source,
 };
-use crate::routing::tests::test_types::{PlanRouteErrorTestCase, PlanRouteTestCase};
+use crate::routing::tests::test_types::{
+    PlanRouteErrorTestCase, PlanRouteReasonTestCase, PlanRouteTestCase,
+};
 
 const HMAC: &str = "email masked with hmac, whose key must never appear in SQL";
 const SMALL: Option = Some(1024 * 1024);
@@ -369,3 +371,30 @@ fn given_an_impossible_combination_when_planning_then_the_error_says_why() {
         );
     }
 }
+
+#[test]
+fn given_files_sqlload_must_read_when_planning_then_the_route_says_why() {
+    let test_cases = [PlanRouteReasonTestCase {
+        description: "Parquet in S3 Databricks can't load exactly, outside every external location",
+        reader: ReaderOffer {
+            direct_blocked: Some("at is a time".to_owned()),
+            ..files_source("s3://other/scores/", FileFormat::Parquet)
+        },
+        writer: databricks_destination(),
+        expected_reason: "SQLLoad reads the files because at is a time",
+    }];
+    for test_case in test_cases {
+        let reason = plan_route(
+            &test_case.reader,
+            &test_case.writer,
+            RouteOverrides::default(),
+        )
+        .map(|route| route.reason)
+        .unwrap_or_default();
+        assert!(
+            reason.starts_with(test_case.expected_reason),
+            "{}: {reason}",
+            test_case.description
+        );
+    }
+}
diff --git a/crates/sqlload/src/routing/tests/test_types.rs b/crates/sqlload/src/routing/tests/test_types.rs
index 708710423..b9b770392 100644
--- a/crates/sqlload/src/routing/tests/test_types.rs
+++ b/crates/sqlload/src/routing/tests/test_types.rs
@@ -16,3 +16,11 @@ pub(crate) struct PlanRouteErrorTestCase {
     pub(crate) overrides: RouteOverrides,
     pub(crate) expected_message: &'static str,
 }
+
+pub(crate) struct PlanRouteReasonTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) reader: ReaderOffer,
+    pub(crate) writer: WriterAccepts,
+    /// How the route's reason starts.
+    pub(crate) expected_reason: &'static str,
+}
diff --git a/website/src/content/docs/docs/targets/databricks.mdx b/website/src/content/docs/docs/targets/databricks.mdx
index 88f3a1215..d9eeca586 100644
--- a/website/src/content/docs/docs/targets/databricks.mdx
+++ b/website/src/content/docs/docs/targets/databricks.mdx
@@ -56,7 +56,8 @@ is still filling is kept.
 
 Parquet files of a files source in S3 load straight from the bucket, with no row passing through
 SQLLoad, when a Unity Catalog external location covers their folder. `plan` lists the external
-locations the service principal can see, which needs `READ FILES` on the one it reads through:
+locations the service principal can see, and checks with `LIST` that it can read a folder holding
+files to load, which needs `READ FILES` on the location it reads through:
 
 ```sql
 CREATE EXTERNAL LOCATION lake URL 's3://example-bucket/events/' WITH (STORAGE CREDENTIAL lake_reader);
@@ -67,17 +68,23 @@ GRANT READ FILES ON EXTERNAL LOCATION lake TO `
Date: Wed, 18 Nov 2026 19:58:21 +0000
Subject: [PATCH 4/4] wip: Databricks S3 route follow-up: undo covers a COPY
 whose count fails to parse, columns grouped case-insensitively, conditional
 READ FILES fix, batch atomicity in docs

---
 .../databricks/_helpers/write/direct_load.rs        |  6 ++++--
 .../adapters/databricks/_helpers/write/s3_copy.rs   | 11 ++++++++---
 .../src/adapters/databricks/tests/test_s3_copy.rs   |  9 +++++++++
 .../src/adapters/dispatch/main/route/list_reach.rs  |  2 +-
 .../src/content/docs/docs/targets/databricks.mdx    | 13 +++++++------
 5 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs
index 2ba363864..805a3e705 100644
--- a/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs
+++ b/crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs
@@ -69,7 +69,10 @@ impl DatabricksWriter {
         for group in copy_groups(schemas, MAX_FILES_PER_COPY) {
             let statement = group_statement(batch, &group, (into, columns), folder.clone())?;
             let result = match execute_statement(&self.destination.api, &statement).await {
-                Ok(result) => inserted_rows(&result),
+                Ok(result) => {
+                    copied = true;
+                    inserted_rows(&result)
+                }
                 Err(error) => Err(error.into()),
             };
             match result {
@@ -77,7 +80,6 @@ impl DatabricksWriter {
                 Err(error) if copied => return Err(self.undo(batch, error).await),
                 Err(error) => return Err(error),
             }
-            copied = true;
         }
         let expected = footer_rows(batch.footers);
         if inserted != expected {
diff --git a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs
index 606f00984..ea13bf884 100644
--- a/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs
+++ b/crates/sqlload/src/adapters/databricks/_helpers/write/s3_copy.rs
@@ -117,7 +117,10 @@ pub(crate) fn copy_groups(files: Vec<(T, &Schema)>, limit: usize) -> Vec(files: Vec<(T, &Schema)>, limit: usize) -> Vec bool {
     for field in schema.fields() {
         for other in held {
-            if other.name() == field.name() && other.data_type() != field.data_type() {
+            if other.name().eq_ignore_ascii_case(field.name())
+                && other.data_type() != field.data_type()
+            {
                 return false;
             }
         }
diff --git a/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs
index f6552d3b8..ce94148de 100644
--- a/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs
+++ b/crates/sqlload/src/adapters/databricks/tests/test_s3_copy.rs
@@ -55,6 +55,15 @@ fn given_a_batchs_parquet_files_when_databricks_copies_them_then_each_copy_holds
             limit: 1000,
             expected_groups: vec![vec!["a", "c"], vec!["b", "d"]],
         },
+        CopyGroupsTestCase {
+            description: "one column named in different cases is one column to Databricks, so different types still split",
+            files: vec![
+                ("a", vec![("ID", DataType::Int32)]),
+                ("b", vec![("id", DataType::Int64)]),
+            ],
+            limit: 1000,
+            expected_groups: vec![vec!["a"], vec!["b"]],
+        },
         CopyGroupsTestCase {
             description: "no copy holds more files than the limit",
             files: vec![
diff --git a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs
index b6c4cb5c2..2b741f084 100644
--- a/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs
+++ b/crates/sqlload/src/adapters/dispatch/main/route/list_reach.rs
@@ -25,7 +25,7 @@ pub async fn list_reach(destination: &Destination, folder: Option<&str>) -> Resu
         let error = error.to_string();
         let first_line = error.lines().next().unwrap_or_default();
         format!(
-            "{folder} cannot be read by Databricks through the external location at {url}: {}. Fix: grant the service principal READ FILES on that external location",
+            "{folder} cannot be read by Databricks through the external location at {url}: {}. Fix: when Databricks refuses access, grant the service principal READ FILES on that external location",
             first_line.trim_end_matches([':', ' '])
         )
     })
diff --git a/website/src/content/docs/docs/targets/databricks.mdx b/website/src/content/docs/docs/targets/databricks.mdx
index d9eeca586..efe8be369 100644
--- a/website/src/content/docs/docs/targets/databricks.mdx
+++ b/website/src/content/docs/docs/targets/databricks.mdx
@@ -68,15 +68,16 @@ GRANT READ FILES ON EXTERNAL LOCATION lake TO `