Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
//! Whether an S3 store points at a custom endpoint, which a warehouse reading S3 itself would not follow.

use crate::store_paths::models::StoreKind;

/// Why an S3 store points somewhere a warehouse reading S3 itself would not read: an endpoint in the connection or in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows.
pub(in crate::adapters) fn custom_endpoint(
store: &StoreKind,
variable: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
let from_env = ["AWS_ENDPOINT", "AWS_ENDPOINT_URL"]
.iter()
.any(|name| variable(name).is_some_and(|value| !value.is_empty()));
match store {
StoreKind::S3 {
endpoint: Some(_), ..
} => Some("the S3 connection sets a custom endpoint".to_owned()),
StoreKind::S3 { .. } if from_env => {
Some("AWS_ENDPOINT or AWS_ENDPOINT_URL sets a custom S3 endpoint".to_owned())
}
_ => None,
}
}
1 change: 1 addition & 0 deletions crates/sqlload/src/adapters/contract/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub(in crate::adapters) mod _batch_direct_problem;
pub(in crate::adapters) mod _custom_endpoint;
pub(in crate::adapters) mod _dropped_ordinals;
pub(in crate::adapters) mod _files_direct_problem;
pub(in crate::adapters) mod _footer_rows;
Expand Down
13 changes: 13 additions & 0 deletions crates/sqlload/src/adapters/databricks/_helpers/stage/locations.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
//! How SQLLoad compares folders with Databricks' external locations.

/// An external location's `url` as SQLLoad compares folders with it: `s3://` for the `s3a://` and `s3n://` it may be stored as, ending in `/` so that it only covers what lies below it.
pub(crate) fn location_url(url: &str) -> String {
let url = url.trim_end_matches('/');
let s3 = ["s3a://", "s3n://"]
.iter()
.find_map(|scheme| url.strip_prefix(scheme));
match s3 {
Some(rest) => format!("s3://{rest}/"),
None => format!("{url}/"),
}
}
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pub(crate) mod folders;
pub(crate) mod locations;
Original file line number Diff line number Diff line change
Expand Up @@ -9,28 +9,18 @@ use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::databricks::constants::COPY_OPTIONS;
use crate::adapters::databricks::models::DatabricksSql;
use crate::column_types::models::ColumnType;
use crate::schema_plan::models::DesiredColumn;

/// `COPY INTO` the batch's table (the staging or build table, or the table itself) from `folder`, each column cast to the type it lands as: the table's own types straight into it or a build table, JSON parsed; text for JSON and UUIDs in a staging table.
/// `COPY INTO` the batch's table (the staging or build table, or the table itself) from `folder`, each column cast to the type it lands as (see [`typed_value`]).
pub(crate) fn copy_statement(target: &WriteTarget<'_>, folder: &str) -> String {
let adapter = DatabricksSql;
let table = target.staging.unwrap_or(target.table);
let final_types = target.direct || target.builds;
let selected: Vec<String> = target
.columns
.iter()
.map(|column| {
let name = adapter.render_identifier(&column.name);
match (final_types, &column.column_type) {
(true, ColumnType::Json) => format!("parse_json({name}) AS {name}"),
(true, column_type) => format!(
"CAST({name} AS {}) AS {name}",
adapter.render_loader_logical_type(column_type).sql
),
(false, column_type) => format!(
"CAST({name} AS {}) AS {name}",
adapter.render_staging_type(column_type)
),
}
format!("{} AS {name}", typed_value(target, column, &name))
})
.collect();
format!(
Expand All @@ -41,6 +31,22 @@ pub(crate) fn copy_statement(target: &WriteTarget<'_>, folder: &str) -> String {
)
}

/// `value` as `column` lands in `target`'s batch table: in the table's own type straight into it or a build table, JSON parsed; in a staging table, text for JSON and UUIDs.
pub(crate) fn typed_value(target: &WriteTarget<'_>, column: &DesiredColumn, value: &str) -> String {
let adapter = DatabricksSql;
match (target.direct || target.builds, &column.column_type) {
(true, ColumnType::Json) => format!("parse_json({value})"),
(true, column_type) => format!(
"CAST({value} AS {})",
adapter.render_loader_logical_type(column_type).sql
),
(false, column_type) => format!(
"CAST({value} AS {})",
adapter.render_staging_type(column_type)
),
}
}

/// `batch` with its time-of-day columns as text, since Databricks reads no Parquet `TIME` and stores times as text.
pub(crate) fn readable_batch(batch: RecordBatch) -> Result<RecordBatch, String> {
let schema = batch.schema();
Expand Down
227 changes: 227 additions & 0 deletions crates/sqlload/src/adapters/databricks/_helpers/write/direct_load.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
//! A files batch loaded by Databricks `COPY`ing the source's own Parquet files from S3.

use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem;
use crate::adapters::contract::main::_footer_rows::footer_rows;
use crate::adapters::contract::models::{DirectBatch, DirectLoaded, TableName};
use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::databricks::_helpers::write::s3_copy::{
S3Batch, S3File, copy_groups, s3_copy_statement,
};
use crate::adapters::databricks::constants::{INSERTED_ROWS_COLUMN, MAX_FILES_PER_COPY};
use crate::adapters::databricks::main::stage::_direct_problem::direct_problem;
use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter};
use crate::databricks_api::main::execute_statement::execute_statement;
use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN};
use crate::files_source::main::normal_value::normal_value;
use crate::files_source::models::DataFile;
use crate::load_naming::main::metadata_name::metadata_name;
use crate::load_naming::models::MetadataRole;
use crate::loader::errors::LoaderError;
use crate::schema_plan::models::DesiredColumn;
use crate::sql_api::models::SqlRows;
use arrow_schema::Schema;

impl DatabricksWriter {
/// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, into the staging or build table the load publishes from, or into the table itself: one `COPY INTO` (one transaction) per [`copy_groups`] group, as `FILES` takes at most 1000 files and `mergeSchema` won't merge one column held as two types. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose inserted rows, as `COPY INTO` reports them, differ from its footers'; an append into the table itself first takes back what it committed. Files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location.
pub(in crate::adapters) async fn copy_files(
&mut self,
batch: &DirectBatch<'_>,
) -> Result<Option<DirectLoaded>, LoaderError> {
let Some(source) = batch.spec.files.as_ref() else {
return Ok(None);
};
let folder = source.location.to_string();
let covered = self
.destination
.external_locations
.get()
.is_some_and(|locations| locations.iter().any(|url| folder.starts_with(url.as_str())));
if !covered {
return Ok(None);
}
if let Some(problem) =
batch_direct_problem(batch, source, direct_problem(batch.spec, batch.read))
{
return Err(LoaderError::Invalid(format!(
"Databricks cannot load this batch's files exactly by reading them itself, and they never go through SQLLoad instead: {problem}"
)));
}
let target = &batch.target;
let (into, columns) = match (target.staging, target.builds) {
(Some(staging), false) => (staging, target.columns),
(staging, _) => (staging.unwrap_or(target.table), batch.write_columns),
};
let no_footer = Schema::empty();
let schemas: Vec<(&DataFile, &Schema)> = batch
.files
.iter()
.map(|data| {
let schema = batch
.footers
.parquet
.get(&data.file.store_key())
.map_or(&no_footer, |footer| footer.schema().as_ref());
(data, schema)
})
.collect();
let mut inserted = 0;
let mut copied = false;
for group in copy_groups(schemas, MAX_FILES_PER_COPY) {
let statement = group_statement(batch, &group, (into, columns), folder.clone())?;
let result = match execute_statement(&self.destination.api, &statement).await {
Ok(result) => {
copied = true;
inserted_rows(&result)
}
Err(error) => Err(error.into()),
};
match result {
Ok(rows) => inserted += rows,
Err(error) if copied => return Err(self.undo(batch, error).await),
Err(error) => return Err(error),
}
}
let expected = footer_rows(batch.footers);
if inserted != expected {
let error = LoaderError::Invalid(format!(
"Databricks loaded {inserted} rows from this batch's files, but their footers hold {expected}"
));
return Err(self.undo(batch, error).await);
}
Ok(Some(match target.staging {
Some(_) => DirectLoaded::Staged(expected),
None => DirectLoaded::Committed(expected),
}))
}
}

impl DatabricksWriter {
/// After `error`, take back the rows a batch going straight into the table committed: its load's rows from its files, which a later run would otherwise skip as loaded. A batch landing in a staging table needs nothing, as the load drops it.
async fn undo(&self, batch: &DirectBatch<'_>, error: LoaderError) -> LoaderError {
let target = &batch.target;
if target.staging.is_some() {
return error;
}
let Some(source) = batch.spec.files.as_ref() else {
return error;
};
let literal = |text: &str| DatabricksSql.render_loader_value_literal(text);
let paths: Vec<String> = batch
.files
.iter()
.map(|data| literal(&format!("{}{}", source.location, data.file.relative_key)))
.collect();
let statement = format!(
"DELETE FROM {} WHERE {} = {} AND {} IN ({})",
DatabricksSql.render_qualified_name(target.table),
DatabricksSql.render_identifier(metadata_name(target.compat, MetadataRole::LoadId)),
literal(target.load_id),
DatabricksSql.render_identifier(SOURCE_FILE_COLUMN),
paths.join(", ")
);
match execute_statement(&self.destination.api, &statement).await {
Ok(_) => error,
Err(undo) => LoaderError::Invalid(format!(
"{error}; taking back the rows it committed then failed too, so the table holds them: {undo}"
)),
}
}
}

/// The `COPY INTO` that loads `group`, files of `batch` below `folder`, into `into`'s `columns`, reading each table column the group's files hold by its name in them.
fn group_statement(
batch: &DirectBatch<'_>,
group: &[&DataFile],
(into, columns): (&TableName, &[DesiredColumn]),
folder: String,
) -> Result<String, LoaderError> {
let mut files = Vec::with_capacity(group.len());
let mut in_files: Vec<(String, String)> = Vec::new();
for data in group {
files.push(s3_file(batch, data, columns)?);
let Some(footer) = batch.footers.parquet.get(&data.file.store_key()) else {
continue;
};
for field in footer.schema().fields() {
let loaded = batch.spec.source_table.loaded_as(field.name()).to_owned();
if !in_files.iter().any(|(name, _)| *name == loaded) {
in_files.push((loaded, field.name().clone()));
}
}
}
Ok(s3_copy_statement(
(&DatabricksSql.render_qualified_name(into), columns),
&batch.target,
&S3Batch {
folder,
files,
in_files,
},
batch.loaded_at,
))
}

/// The rows a `COPY INTO`'s `result` reports it inserted.
fn inserted_rows(result: &SqlRows) -> Result<u64, LoaderError> {
let reported = result
.columns
.iter()
.position(|column| column == INSERTED_ROWS_COLUMN)
.and_then(|index| result.rows.first()?.get(index)?.clone());
let Some(text) = reported else {
return Err(LoaderError::Invalid(format!(
"COPY INTO reported no {INSERTED_ROWS_COLUMN}"
)));
};
text.parse::<u64>().map_err(|error| {
LoaderError::Invalid(format!(
"COPY INTO reported {INSERTED_ROWS_COLUMN} = {text:?}: {error}"
))
})
}

/// The values SQLLoad gives every row of `data`: its name and version as the files source records them, and its partition values (constants among them), each converted to its column's type as SQLLoad's reader would, so a value that does not convert fails here rather than in Databricks' cast.
fn s3_file(
batch: &DirectBatch<'_>,
data: &DataFile,
columns: &[DesiredColumn],
) -> Result<S3File, LoaderError> {
let Some(source) = batch.spec.files.as_ref() else {
return Err(LoaderError::Invalid("not a files table".to_owned()));
};
let mut constants = vec![
(
SOURCE_FILE_COLUMN.to_owned(),
Some(format!("{}{}", source.location, data.file.relative_key)),
),
(SOURCE_VERSION_COLUMN.to_owned(), Some(data.file.version())),
];
for (name, _) in &source.partition_types {
let value = data
.partitions
.iter()
.find(|(partition, _)| partition == name)
.map(|(_, value)| value.clone());
let loaded = batch.spec.source_table.loaded_as(name);
let column_type = columns
.iter()
.find(|column| column.name == loaded)
.map(|column| &column.column_type);
let value = match (value, column_type) {
(Some(value), Some(column_type)) => {
Some(normal_value(&value, column_type).map_err(|reason| {
LoaderError::Invalid(format!(
"{}: {name} = {value:?} does not load as {column_type}: {reason}",
data.file.relative_key
))
})?)
}
(value, _) => value,
};
constants.push((loaded.to_owned(), value));
}
Ok(S3File {
key: data.file.relative_key.clone(),
constants,
})
}
2 changes: 2 additions & 0 deletions crates/sqlload/src/adapters/databricks/_helpers/write/mod.rs
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
pub(crate) mod copy_select;
pub(crate) mod direct_load;
pub(crate) mod s3_copy;
pub(crate) mod table_writer;
Loading
Loading