Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions crates/sqlload/src/adapters/contract/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,8 @@ pub const COMMIT_STATEMENT: &str = "COMMIT";

/// Ends a transaction, undoing it.
pub const ROLLBACK_STATEMENT: &str = "ROLLBACK";

/// The `_sqlload_row` span of one unloaded batch (2^48): its files' rows number from the batch times this.
pub const UNLOADED_BATCH_ROWS: i64 = 1 << 48;
/// The `_sqlload_row` span of one unloaded file in a batch (2^32).
pub const UNLOADED_FILE_ROWS: i64 = 1 << 32;
15 changes: 12 additions & 3 deletions crates/sqlload/src/adapters/contract/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -283,11 +283,11 @@ pub struct DirectBatch<'a> {
pub source_folder: Option<&'a OpenFolder>,
}

/// One unload's files the destination loads by reading them itself: times and timestamps held as whole microseconds, every other column as the type it is read as.
/// A warehouse source's rows the destination loads itself, without SQLLoad seeing them: times and timestamps held as whole microseconds, every other column as the type it is read as.
#[derive(Debug, Clone, Copy)]
pub struct UnloadedBatch<'a> {
pub files: &'a UnloadedFiles,
/// The columns the files hold, as they are read.
pub source: UnloadedSource<'a>,
/// The columns the rows hold, as they are read.
pub read: &'a [ReadColumn],
/// The table's columns as the load writes them.
pub write_columns: &'a [DesiredColumn],
Expand All @@ -296,6 +296,15 @@ pub struct UnloadedBatch<'a> {
pub loaded_at: &'a str,
}

/// Where the destination reads an unloaded batch's rows.
#[derive(Debug, Clone, Copy)]
pub enum UnloadedSource<'a> {
/// One unload's files, which it reads itself.
Files(&'a UnloadedFiles),
/// The unload's query, which it runs itself in the source's own session.
Query(&'a str),
}

/// A query a check before publish runs on a relation; each returns its values as text.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CheckQuery<'a> {
Expand Down
37 changes: 30 additions & 7 deletions crates/sqlload/src/adapters/dispatch/main/route/unload_blocked.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,37 @@
//! Why the destination cannot load a warehouse source's unloaded files exactly by reading them itself.

use crate::adapters::snowflake::main::copy::_stage_covers::stage_covers;
use crate::adapters::snowflake::models::StageRoute;
use crate::column_types::models::Warehouse;
use crate::loader::constants::UNLOADED_TIMES;
use crate::loader::models::TableSpec;
use crate::loader::constants::{UNLOADED_OUTSIDE_STAGE, UNLOADED_TIMES};
use crate::loader::models::WarehouseRoute;
use crate::routing::models::Store;
use crate::store_paths::main::resolve_location::resolve_location;

/// Why the destination cannot load the table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB turns the files' whole microseconds back into times and timestamps; masks need SQLLoad to see the rows.
pub fn unload_blocked(spec: &TableSpec) -> Option<String> {
match (spec.warehouse, spec.masked.columns.is_empty()) {
(Warehouse::DuckDb, true) => None,
(Warehouse::DuckDb, false) => Some("the table masks columns".to_owned()),
/// Why `warehouse` cannot load a table's unloaded files exactly by reading them itself, so SQLLoad reads them instead; `None` when it can (where it also reaches their store). Only DuckDB and Snowflake turn the files' whole microseconds back into times and timestamps; a table that `masks` columns needs SQLLoad to see the rows; Snowflake reads another session's unload only through the target's external `stage`, which must cover the S3 folder `route` unloads to, as a user or named stage of the source's is not its to read.
pub fn unload_blocked(
warehouse: Warehouse,
masks: bool,
route: Option<&WarehouseRoute>,
stage: Option<&StageRoute>,
) -> Option<String> {
let covered = match (route, stage) {
(
Some(WarehouseRoute::Unload {
store: Store::ObjectStore(url),
..
}),
Some(StageRoute::External(stage)),
) => resolve_location(None, &format!("{}/", url.trim_end_matches('/')))
.is_ok_and(|unloaded| stage_covers(&stage.location, &unloaded)),
_ => false,
};
match (warehouse, masks) {
(Warehouse::DuckDb | Warehouse::Snowflake, true) => {
Some("the table masks columns".to_owned())
}
(Warehouse::Snowflake, false) if !covered => Some(UNLOADED_OUTSIDE_STAGE.to_owned()),
(Warehouse::DuckDb | Warehouse::Snowflake, false) => None,
_ => Some(UNLOADED_TIMES.to_owned()),
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,10 @@ pub async fn load_unloaded(
writer: &mut Writer,
batch: &UnloadedBatch<'_>,
) -> Result<Option<DirectLoaded>, LoaderError> {
#[cfg(not(feature = "duckdb"))]
let _ = batch;
match writer {
#[cfg(feature = "duckdb")]
Writer::DuckDb(duckdb) => duckdb.pull_unloaded(batch).await.map(Some),
Writer::Snowflake(snowflake) => snowflake.pull_unloaded(batch).await.map(Some),
_ => Ok(None),
}
}
3 changes: 2 additions & 1 deletion crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
//! The statement that has DuckDB load Parquet files itself.

use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS};
use crate::adapters::contract::models::TableName;
use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::duckdb::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS, UTC_OFFSET};
use crate::adapters::duckdb::constants::UTC_OFFSET;
use crate::adapters::duckdb::models::DuckDbSql;
use crate::column_types::main::is_variant::is_variant;
use crate::column_types::models::ColumnType;
Expand Down
8 changes: 6 additions & 2 deletions crates/sqlload/src/adapters/duckdb/_helpers/unloaded_load.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! A warehouse source's unloaded files loaded by DuckDB reading them itself.

use crate::adapters::contract::constants::{BEGIN_STATEMENT, COMMIT_STATEMENT};
use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch};
use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource};
use crate::adapters::duckdb::_helpers::pull_sql::{UnloadedColumns, unloaded_insert};
use crate::adapters::duckdb::main::_execute_statements::execute_statements;
use crate::adapters::duckdb::main::_pull_files::pull_files;
Expand All @@ -19,7 +19,11 @@ impl DuckDbWriter {
true => DirectLoaded::Committed(rows),
false => DirectLoaded::Staged(rows),
};
let files = batch.files;
let UnloadedSource::Files(files) = batch.source else {
return Err(LoaderError::Invalid(
"DuckDB reads a warehouse source's rows only from its unloaded files".to_owned(),
));
};
if files.keys.is_empty() {
return Ok(loaded(0));
}
Expand Down
4 changes: 0 additions & 4 deletions crates/sqlload/src/adapters/duckdb/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,6 @@ pub const MOTHERDUCK_TOKEN_VARIABLES: &[&str] = &["motherduck_token", "MOTHERDUC
pub const MOTHERDUCK_UNSUPPORTED_DUCKDB: &str = "is not yet supported by MotherDuck";
/// What precedes the extension's own message in DuckDB's error.
pub const EXTENSION_EXCEPTION: &str = "threw an exception: ";
/// The `_sqlload_row` span of one unloaded batch (2^48): its files' rows number from the batch times this.
pub const UNLOADED_BATCH_ROWS: i64 = 1 << 48;
/// The `_sqlload_row` span of one unloaded file in a batch (2^32).
pub const UNLOADED_FILE_ROWS: i64 = 1 << 32;
/// MotherDuck's server version, such as `v1.5.6-2026-11-382`.
pub const MOTHERDUCK_VERSION_QUERY: &str = "SELECT md_version()";
/// The first DuckDB major version that shreds a VARIANT column's common fields into columns, so queries on them are fast.
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlload/src/adapters/duckdb/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ pub(crate) fn pulled_unloaded(reported: u64, with_files: bool) -> Result<(String
table: "loaded".to_owned(),
};
let batch = UnloadedBatch {
files: &files,
source: crate::adapters::contract::models::UnloadedSource::Files(&files),
read: &read,
write_columns: &columns,
target: WriteTarget {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@ pub(crate) mod copy_select;
pub(crate) mod sql_adapter;
pub(crate) mod text_values;
pub(crate) mod type_names;
pub(crate) mod unloaded_values;
pub(crate) mod values;
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
//! The value each column of a Snowflake source's batch loaded inside Snowflake takes, from the source's unloaded files or its query.

use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS};
use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::snowflake::models::{SnowflakeSql, UnloadedFrom, UnloadedLoad, UnloadedRows};
use crate::column_types::models::ColumnType;
use crate::load_naming::main::metadata_role::metadata_role;
use crate::load_naming::models::MetadataRole;
use crate::schema_plan::constants::MERGE_ROW_COLUMN;
use crate::schema_plan::models::DesiredColumn;

/// Where `column` comes from in a statement reading `rows`: the batch's load id or load time, a merge's row number (the batch, then the file, then the row in it; a query's rows in the order it gives them), the value of the constant it is, or the source's column it is read as; NULL when the source lacks it.
pub(crate) fn column_value(
request: &UnloadedLoad<'_>,
column: &DesiredColumn,
rows: &UnloadedRows<'_>,
) -> String {
let literal = |text: &str| SnowflakeSql.render_loader_value_literal(text);
let constant = request
.constants
.iter()
.find(|(name, _, _)| *name == column.name);
match (metadata_role(request.compat, &column.name), constant) {
(Some(MetadataRole::LoadId), _) => literal(request.load_id),
(Some(MetadataRole::LoadedAt), _) => {
format!("CAST({} AS TIMESTAMP_TZ(6))", literal(request.loaded_at))
}
_ if column.name == MERGE_ROW_COLUMN => {
let in_batch = match (rows, &request.from) {
(
UnloadedRows::Files(first, keys),
UnloadedFrom::Files {
filename_prefix, ..
},
) => {
let files: Vec<String> = keys
.iter()
.enumerate()
.map(|(index, key)| {
format!(
"WHEN {} THEN {}",
literal(&format!("{filename_prefix}{key}")),
first + index
)
})
.collect();
format!(
"(CASE METADATA$FILENAME {} ELSE TO_NUMBER('sqlload: unexpected unloaded file ' || METADATA$FILENAME) END) * {UNLOADED_FILE_ROWS} + METADATA$FILE_ROW_NUMBER",
files.join(" ")
)
}
_ => "SEQ8()".to_owned(),
};
format!("{} * {UNLOADED_BATCH_ROWS} + {in_batch}", request.batch)
}
(_, Some((_, _, value))) => cast(column, &literal(value)),
_ => request
.read
.iter()
.find(|read| read.name == column.name)
.map_or_else(
|| "NULL".to_owned(),
|read| {
let field = match rows {
UnloadedRows::Files(..) => format!("GET($1, {})", literal(&read.name)),
UnloadedRows::Query => format!("u.{}", quoted(&read.name)),
};
unloaded_value(column, &read.output, &field)
},
),
}
}

/// `name` as the unload's query names its column: quoted exactly as written.
fn quoted(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}

/// An unloaded `field`, read as `read`, as the value it was unloaded from, then as `column` holds it: a time or timestamp from its whole microseconds (a timestamp with time zone in UTC, as SQLLoad's reader gives it), anything else as it is.
fn unloaded_value(column: &DesiredColumn, read: &ColumnType, field: &str) -> String {
let micros = format!("CAST({field} AS NUMBER(38,0))");
let value = match read {
ColumnType::TimestampNtz => format!("TO_TIMESTAMP_NTZ({micros}, 6)"),
ColumnType::TimestampTz => {
format!("CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_TZ({micros}, 6))")
}
ColumnType::Time => format!("TIMEADD(MICROSECOND, {micros}, '00:00:00'::TIME)"),
_ => field.to_owned(),
};
cast(column, &value)
}

/// `value` as `column` holds it: JSON and arrays parsed from their text, anything else cast to its type.
fn cast(column: &DesiredColumn, value: &str) -> String {
match &column.column_type {
ColumnType::Json => format!("PARSE_JSON(CAST({value} AS VARCHAR))"),
ColumnType::Array(_) => format!("CAST(PARSE_JSON(CAST({value} AS VARCHAR)) AS ARRAY)"),
column_type => format!(
"CAST({value} AS {})",
SnowflakeSql.render_loader_logical_type(column_type).sql
),
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ fn held_types(columns: &[DesiredColumn]) -> Vec<DesiredColumn> {
}

/// The columns a staging table holds, as the staged rows carry them: JSON and UUIDs as text; also the columns that write a table of `held_types` on with no cast.
fn staged_as_text(columns: &[DesiredColumn]) -> Vec<DesiredColumn> {
pub(in crate::adapters) fn staged_as_text(columns: &[DesiredColumn]) -> Vec<DesiredColumn> {
columns
.iter()
.cloned()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ pub(crate) mod file_keys;
pub(crate) mod row_stream;
pub(crate) mod rows_loaded;
pub(crate) mod table_writer;
pub(crate) mod unloaded_load;
Loading
Loading