diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs index 36324ece3..1fe987921 100644 --- a/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs +++ b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod pull; pub(crate) mod sql_adapter; pub(crate) mod type_names; pub(crate) mod write; diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/dialect/pull.rs b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/pull.rs new file mode 100644 index 000000000..3704a21a6 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/pull.rs @@ -0,0 +1,245 @@ +//! The query that has BigQuery load Parquet files in GCS itself. + +use serde_json::{Map, Value, json}; + +use crate::adapters::bigquery::constants::{ + DECIMAL_TARGET_TYPES, MAX_COLUMN_NAME, MAX_FILES_PER_QUERY, MAX_QUERY_BYTES, + RESERVED_COLUMN_PREFIXES, UNEXPECTED_FILE, +}; +use crate::adapters::bigquery::models::BigQuerySql; +use crate::adapters::contract::models::TableName; +use crate::adapters::contract::types::SqlAdapter; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::{Compat, MetadataRole}; +use crate::schema_plan::models::DesiredColumn; + +/// One file BigQuery reads: its `gs://` URL, as `_FILE_NAME` reports it, and the values SQLLoad gives every row of it (its name and version as the files source records them, and its partition values, `None` for a partition folder it lacks), by column. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PulledFile { + pub url: String, + pub constants: Vec<(String, Option)>, +} + +/// Files BigQuery reads through one temporary external table, which takes one file's schema for all of its files: files whose footers hold the same columns, and the table's columns they hold, each with its name in the files. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PulledGroup { + pub files: Vec, + pub in_files: Vec<(String, String)>, +} + +/// The `INSERT INTO into` of every one of `columns` from the Parquet files of `groups`, and the `tableDefinitions` of the query job running it: one temporary external table per group, its rows appended to the others'. A column comes from the row's file's constants when the files have one of that name (picked by `_FILE_NAME`), otherwise from the group's files when they hold it, and is cast to its type in a build table (`builds`), else in the staging table, without its parameters (`NUMERIC(12,2)` as `NUMERIC`), which a cast can't name and the column then checks; the load id and load time are the batch's; any other column is NULL. +pub(crate) fn pull_query( + into: &TableName, + (columns, builds): (&[DesiredColumn], bool), + groups: &[PulledGroup], + (compat, load_id, loaded_at): (Compat, &str, &str), +) -> (String, Value) { + let selects: Vec = groups + .iter() + .enumerate() + .map(|(index, group)| { + group_select( + index, + group, + (columns, builds), + (compat, load_id, loaded_at), + ) + }) + .collect(); + let targets: Vec = columns + .iter() + .map(|column| BigQuerySql.render_identifier(&column.name)) + .collect(); + let query = format!( + "INSERT INTO {} ({}) {}", + BigQuerySql.render_qualified_name(into), + targets.join(", "), + selects.join(" UNION ALL ") + ); + let mut definitions = Map::new(); + for (index, group) in groups.iter().enumerate() { + let urls: Vec<&str> = group.files.iter().map(|file| file.url.as_str()).collect(); + definitions.insert(group_table(index), table_definition(&urls)); + } + (query, Value::Object(definitions)) +} + +/// [`pull_query`] over `groups`, split into as many queries as keep each under [`MAX_QUERY_BYTES`] and [`MAX_FILES_PER_QUERY`] files, halving the files until they fit; a single file whose query is still too long is one query, which BigQuery refuses. +pub(crate) fn pull_queries( + into: &TableName, + (columns, builds): (&[DesiredColumn], bool), + groups: &[PulledGroup], + (compat, load_id, loaded_at): (Compat, &str, &str), +) -> Vec<(String, Value)> { + let mut queries: Vec<(String, Value)> = Vec::new(); + let mut pending: Vec> = vec![groups.to_vec()]; + while let Some(part) = pending.pop() { + let files: usize = part.iter().map(|group| group.files.len()).sum(); + let (query, definitions) = + pull_query(into, (columns, builds), &part, (compat, load_id, loaded_at)); + let over = query.len() > MAX_QUERY_BYTES || files > MAX_FILES_PER_QUERY; + match (over, files) { + (true, count) if count > 1 => { + let (first, second) = split_groups(&part, count / 2); + pending.push(second); + pending.push(first); + } + _ => queries.push((query, definitions)), + } + } + queries +} + +/// `groups` cut after their first `at` files, in order, each side keeping only groups that hold files. +fn split_groups(groups: &[PulledGroup], at: usize) -> (Vec, Vec) { + let mut first: Vec = Vec::new(); + let mut second: Vec = Vec::new(); + let mut left = at; + for group in groups { + let take = left.min(group.files.len()); + left -= take; + let (head, tail) = group.files.split_at(take); + for (side, files) in [(&mut first, head), (&mut second, tail)] { + if !files.is_empty() { + side.push(PulledGroup { + files: files.to_vec(), + in_files: group.in_files.clone(), + }); + } + } + } + (first, second) +} + +/// The `SELECT` of `columns` from the `index`th group's external table, as [`pull_query`] fills them. +fn group_select( + index: usize, + group: &PulledGroup, + (columns, builds): (&[DesiredColumn], bool), + (compat, load_id, loaded_at): (Compat, &str, &str), +) -> String { + let literal = |text: &str| BigQuerySql.render_loader_value_literal(text); + let mut values: Vec = Vec::with_capacity(columns.len()); + for column in columns { + let value = match (metadata_role(compat, &column.name), column.name.as_str()) { + (Some(MetadataRole::LoadId), _) => literal(load_id), + (Some(MetadataRole::LoadedAt), _) => literal(loaded_at), + (_, name) + if group + .files + .first() + .is_some_and(|file| constant(file, name).is_some()) => + { + by_file(&group.files, name) + } + (_, name) if let Some(file) = in_file(&group.in_files, name) => { + format!("f.{}", BigQuerySql.render_exact_identifier(file)) + } + _ => "NULL".to_owned(), + }; + let sql_type = match builds { + true => { + BigQuerySql + .render_loader_logical_type(&column.column_type) + .sql + } + false => BigQuerySql.render_staging_type(&column.column_type), + }; + let base = sql_type + .split_once('(') + .map_or(sql_type.as_str(), |(base, _)| base); + values.push(format!("CAST({value} AS {base})")); + } + format!( + "SELECT {} FROM {} AS f", + values.join(", "), + group_table(index) + ) +} + +/// A temporary external table over the Parquet files at `urls`, its decimals read as the first of BigQuery's types holding their precision and scale. +pub(crate) fn table_definition(urls: &[&str]) -> Value { + json!({ + "sourceFormat": "PARQUET", + "sourceUris": urls, + "decimalTargetTypes": DECIMAL_TARGET_TYPES, + }) +} + +/// The temporary external table of the `index`th group of files. +pub(crate) fn group_table(index: usize) -> String { + format!("sqlload_files_{index}") +} + +/// The name in the files of the column the table holds as `name`, when the files hold it. +fn in_file<'a>(in_files: &'a [(String, String)], name: &str) -> Option<&'a str> { + in_files + .iter() + .find(|(held, _)| held == name) + .map(|(_, file)| file.as_str()) +} + +/// `file`'s constant `name`: `Some(None)` when its folder lacks it, `None` when it has no such constant. +fn constant<'a>(file: &'a PulledFile, name: &str) -> Option> { + file.constants + .iter() + .find(|(constant, _)| constant == name) + .map(|(_, value)| value.as_deref()) +} + +/// The row's file's value of constant `name`, as text: each file's value picked by `_FILE_NAME`, a typed NULL where its folder lacks it, so a column no file has a value of still casts to its type; a row of any other file fails the query, so a file name BigQuery reports differently never loads as NULL. +fn by_file(files: &[PulledFile], name: &str) -> String { + let cases: Vec = files + .iter() + .map(|file| { + let value = constant(file, name).flatten().map_or_else( + || "CAST(NULL AS STRING)".to_owned(), + |value| BigQuerySql.render_loader_value_literal(value), + ); + format!( + "WHEN {} THEN {value}", + BigQuerySql.render_loader_value_literal(&file.url) + ) + }) + .collect(); + format!( + "CASE _FILE_NAME {} ELSE ERROR(CONCAT({}, _FILE_NAME)) END", + cases.join(" "), + BigQuerySql.render_loader_value_literal(UNEXPECTED_FILE) + ) +} + +/// Why BigQuery would not read Parquet columns named `names` under those names: one that is not letters, digits and underscores starting with a letter or underscore, which BigQuery renames (`Order ID` reads as `Order_ID`), starts with a prefix BigQuery reserves, or differs from another only in case; `None` when it reads every one as it is. +pub(crate) fn unread_name<'a>(names: impl Iterator) -> Option { + let mut seen: Vec = Vec::new(); + for name in names { + let folded = name.to_ascii_uppercase(); + let plain = name.len() <= MAX_COLUMN_NAME + && name + .chars() + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && name + .chars() + .all(|character| character.is_ascii_alphanumeric() || character == '_'); + let problem = match ( + plain, + RESERVED_COLUMN_PREFIXES + .iter() + .find(|prefix| folded.starts_with(*prefix)), + seen.contains(&folded), + ) { + (false, _, _) => "BigQuery renames a name that isn't letters, digits and underscores starting with a letter or underscore".to_owned(), + (true, Some(prefix), _) => format!("{prefix} is a prefix BigQuery reserves"), + (true, None, true) => "another column has the same name in another case, and BigQuery's names ignore case".to_owned(), + (true, None, false) => { + seen.push(folded); + continue; + } + }; + return Some(format!( + "BigQuery would not read column {name:?} as it is: {problem}" + )); + } + None +} diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs new file mode 100644 index 000000000..a3a8db791 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs @@ -0,0 +1,202 @@ +//! A files batch loaded by BigQuery querying the source's own Parquet files in GCS. + +use serde_json::{Value, json}; + +use crate::adapters::bigquery::_helpers::dialect::pull::{ + PulledFile, PulledGroup, pull_queries, unread_name, +}; +use crate::adapters::bigquery::main::plan::_direct_problem::direct_problem; +use crate::adapters::bigquery::models::BigQueryWriter; +use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem; +use crate::adapters::contract::main::_footer_rows::footer_rows; +use crate::adapters::contract::models::{DirectBatch, DirectLoaded, TableName}; +use crate::adapters::contract::types::TableWriter; +use crate::bigquery_api::main::load_job::load_job; +use crate::bigquery_api::models::BigQueryClient; +use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN}; +use crate::files_source::main::normal_value::normal_value; +use crate::files_source::models::{BatchFooters, DataFile}; +use crate::loader::errors::LoaderError; +use crate::loader::models::FilesSource; +use crate::schema_plan::models::DesiredColumn; +use crate::store_paths::models::StoreKind; + +impl BigQueryWriter { + /// Load `batch` by having BigQuery query its Parquet files in GCS itself, in query jobs that insert them into the staging or build table the load publishes from, through temporary external tables BigQuery reads with its own identity. A batch whose own footers hold a column a cast would not keep exactly, or a column name BigQuery would change, or whose inserted rows differ from its footers', fails; files never fall back to SQLLoad mid-load. `None` when the table is not a files table in GCS. + pub(in crate::adapters) async fn copy_files( + &mut self, + batch: &DirectBatch<'_>, + ) -> Result, LoaderError> { + let Some(source) = batch.spec.files.as_ref() else { + return Ok(None); + }; + let StoreKind::Gcs { .. } = &source.location.store else { + return Ok(None); + }; + let refused = |problem: String| { + LoaderError::Invalid(format!( + "BigQuery cannot load this batch's files exactly by reading them itself, and they never go through SQLLoad instead: {problem}" + )) + }; + if let Some(problem) = + batch_direct_problem(batch, source, direct_problem(batch.spec, batch.read)) + { + return Err(refused(problem)); + } + self.settle_rows().await?; + let target = &batch.target; + let (into, columns) = match (target.staging, target.builds) { + (Some(staging), false) => (staging, target.columns), + (staging, _) => (staging.unwrap_or(target.table), batch.write_columns), + }; + let into = TableName { + database: Some( + into.database + .clone() + .unwrap_or_else(|| self.destination.project.clone()), + ), + ..into.clone() + }; + let groups = pulled_groups(source, (batch, columns)).map_err(refused)?; + let queries = pull_queries( + &into, + (columns, target.builds), + &groups, + (target.compat, target.load_id, batch.loaded_at), + ); + let mut rows = 0; + for (query, definitions) in queries { + rows += query_files(&self.destination.api, query, definitions).await?; + } + let expected = footer_rows(batch.footers); + if rows != expected { + return Err(LoaderError::Invalid(format!( + "BigQuery loaded {rows} rows from this batch's files, but their footers hold {expected}; the batch is not committed" + ))); + } + Ok(Some(DirectLoaded::Staged(rows))) + } +} + +/// The batch's files in groups whose footers hold the same columns, in the batch's order, each file with the values SQLLoad gives its rows; `Err` names a column name BigQuery would change, or a path value that does not convert to its column's type. +fn pulled_groups( + source: &FilesSource, + (batch, columns): (&DirectBatch<'_>, &[DesiredColumn]), +) -> Result, String> { + let mut groups: Vec<(Vec, PulledGroup)> = Vec::new(); + for data in batch.files { + let fields = footer_fields(batch.footers, data); + if let Some(problem) = unread_name(fields.iter().map(String::as_str)) { + return Err(format!("{}: {problem}", data.file.relative_key)); + } + let file = pulled_file(source, data, (batch, columns))?; + match groups.iter_mut().find(|(held, _)| *held == fields) { + Some((_, group)) => group.files.push(file), + None => { + let in_files: Vec<(String, String)> = fields + .iter() + .map(|field| { + let loaded = batch.spec.source_table.loaded_as(field); + (loaded.to_owned(), field.clone()) + }) + .collect(); + groups.push(( + fields, + PulledGroup { + files: vec![file], + in_files, + }, + )); + } + } + } + Ok(groups.into_iter().map(|(_, group)| group).collect()) +} + +/// What BigQuery reads for `data`: its `gs://` URL, and the values SQLLoad gives every row of it, each path value converted to its column's type as SQLLoad's reader would; `Err` names a value that does not convert. +fn pulled_file( + source: &FilesSource, + data: &DataFile, + (batch, columns): (&DirectBatch<'_>, &[DesiredColumn]), +) -> Result { + let url = format!("{}{}", source.location, data.file.relative_key); + let mut constants = vec![ + (SOURCE_FILE_COLUMN.to_owned(), Some(url.clone())), + (SOURCE_VERSION_COLUMN.to_owned(), Some(data.file.version())), + ]; + for (name, _) in &source.partition_types { + let value = data + .partitions + .iter() + .find(|(partition, _)| partition == name) + .map(|(_, value)| value.clone()); + let loaded = batch.spec.source_table.loaded_as(name); + let column_type = columns + .iter() + .find(|column| column.name == loaded) + .map(|column| &column.column_type); + let value = match (value, column_type) { + (Some(value), Some(column_type)) => { + Some(normal_value(&value, column_type).map_err(|reason| { + format!( + "{}: {name} = {value:?} does not load as {column_type}: {reason}", + data.file.relative_key + ) + })?) + } + (value, _) => value, + }; + constants.push((loaded.to_owned(), value)); + } + Ok(PulledFile { url, constants }) +} + +/// The column names of `data`'s Parquet footer, in order; none when the batch has no footer of it. +fn footer_fields(footers: &BatchFooters, data: &DataFile) -> Vec { + let Some(footer) = footers.parquet.get(&data.file.store_key()) else { + return Vec::new(); + }; + let schema = footer.schema().clone(); + schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect() +} + +/// Run one query job inserting files in GCS, its temporary external tables `definitions`: the rows it inserted. A job BigQuery fails, or whose count is not a number, fails the batch. +async fn query_files( + api: &BigQueryClient, + query: String, + definitions: Value, +) -> Result { + let job = json!({ + "jobReference": { + "projectId": api.project, + "jobId": format!("sqlload_files_{}", uuid::Uuid::new_v4().simple()), + "location": api.location, + }, + "configuration": {"query": { + "query": query, + "useLegacySql": false, + "tableDefinitions": definitions, + }}, + }); + let resource = load_job(api, &job, None).await?; + let error = &resource["status"]["errorResult"]; + if !error.is_null() { + return Err(LoaderError::Invalid(format!( + "BigQuery could not load this batch's files from GCS: {}: {}", + error["reason"].as_str().unwrap_or_default(), + error["message"].as_str().unwrap_or_default() + ))); + } + let affected = resource["statistics"]["query"]["numDmlAffectedRows"] + .as_str() + .unwrap_or_default(); + affected.parse().map_err(|_| { + LoaderError::Invalid(format!( + "BigQuery's query job reported {affected:?} inserted rows, not a count; the batch is not committed" + )) + }) +} diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs b/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs index 57f955fac..4254929b4 100644 --- a/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod arrow_rows; +pub(crate) mod direct_load; pub(crate) mod table_writer; diff --git a/crates/sqlload/src/adapters/bigquery/constants.rs b/crates/sqlload/src/adapters/bigquery/constants.rs index fc99d7edd..8dc7c8527 100644 --- a/crates/sqlload/src/adapters/bigquery/constants.rs +++ b/crates/sqlload/src/adapters/bigquery/constants.rs @@ -24,3 +24,28 @@ pub const STREAMS_PER_BATCH: usize = 4; /// The most bytes one append request carries, under the Storage Write API's 10 MB request limit. pub const APPEND_REQUEST_BYTES: usize = 9 * 1024 * 1024; + +/// The types BigQuery reads a Parquet decimal as, the first that holds its precision and scale: a temporary external table's `decimalTargetTypes`. +pub const DECIMAL_TARGET_TYPES: [&str; 3] = ["NUMERIC", "BIGNUMERIC", "STRING"]; + +/// The longest column name BigQuery takes. +pub const MAX_COLUMN_NAME: usize = 300; + +/// Prefixes BigQuery reserves for its own columns (in upper case; names ignore case), which no column it reads from Parquet may start with. +pub const RESERVED_COLUMN_PREFIXES: [&str; 6] = [ + "_TABLE_", + "_FILE_", + "_PARTITION", + "_ROW_TIMESTAMP", + "__ROOT__", + "_COLIDENTIFIER", +]; + +/// The most bytes of SQL one query job over files in GCS holds, under BigQuery's limit of 1,024K characters for a query before it is resolved. +pub const MAX_QUERY_BYTES: usize = 900 * 1024; + +/// The most files one query job over files in GCS reads. +pub const MAX_FILES_PER_QUERY: usize = 1000; + +/// What a query over files in GCS fails with on a row of a file not in its batch, before the file's name. +pub const UNEXPECTED_FILE: &str = "sqlload: unexpected file "; diff --git a/crates/sqlload/src/adapters/bigquery/main/mod.rs b/crates/sqlload/src/adapters/bigquery/main/mod.rs index f52f1c4a6..6eed751db 100644 --- a/crates/sqlload/src/adapters/bigquery/main/mod.rs +++ b/crates/sqlload/src/adapters/bigquery/main/mod.rs @@ -1 +1,2 @@ +pub(in crate::adapters) mod plan; pub mod session; diff --git a/crates/sqlload/src/adapters/bigquery/main/plan/_check_files.rs b/crates/sqlload/src/adapters/bigquery/main/plan/_check_files.rs new file mode 100644 index 000000000..40cc8e79f --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/main/plan/_check_files.rs @@ -0,0 +1,24 @@ +//! Whether BigQuery reads a Parquet file in GCS itself. + +use serde_json::json; + +use crate::adapters::bigquery::_helpers::dialect::pull::{group_table, table_definition}; +use crate::adapters::bigquery::models::BigQueryDestination; +use crate::bigquery_api::main::query_schema::query_schema; +use crate::http::errors::ApiError; + +/// Check that BigQuery reads the Parquet file at `url` with its own identity (its connection's service account), by a dry run of a query over it as a temporary external table: BigQuery reads the file's schema, which needs `storage.objects.get` on the bucket, and bills nothing. +pub(in crate::adapters) async fn check_files( + destination: &BigQueryDestination, + url: &str, +) -> Result<(), ApiError> { + let table = group_table(0); + let definitions = json!({ table.clone(): table_definition(&[url]) }); + query_schema( + &destination.api, + &format!("SELECT * FROM {table}"), + Some(&definitions), + ) + .await + .map(|_| ()) +} diff --git a/crates/sqlload/src/adapters/bigquery/main/plan/_direct_problem.rs b/crates/sqlload/src/adapters/bigquery/main/plan/_direct_problem.rs new file mode 100644 index 000000000..37c88add7 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/main/plan/_direct_problem.rs @@ -0,0 +1,72 @@ +//! Whether BigQuery may load a files table's files itself. + +use crate::adapters::bigquery::_helpers::dialect::pull::unread_name; +use crate::adapters::contract::main::_custom_endpoint::custom_endpoint; +use crate::adapters::contract::main::_files_direct_problem::files_direct_problem; +use crate::column_types::models::ColumnType; +use crate::files_source::constants::{ + SOURCE_FILE_COLUMN, SOURCE_ROW_COLUMN, SOURCE_VERSION_COLUMN, +}; +use crate::files_source::models::{FilesRead, SourceFormat}; +use crate::loader::models::{FilesSource, TableSpec}; +use crate::store_paths::models::StoreKind; + +/// Why BigQuery cannot load the planned `read` of a files table in GCS exactly by querying its Parquet files where they lie, so SQLLoad reads them instead; `None` when it can, or when the files aren't in GCS. BigQuery reports which file a row came from but not its position in it, so a table writing `_sqlload_source_row` keeps SQLLoad's reader unless it excludes that column. Every column must load as the type the files hold it as; an emulator's endpoint is SQLLoad's to follow; masks need SQLLoad's reader; a column name BigQuery would change keeps it too; and BigQuery reads a Parquet UUID as bytes, a list as a record, an unsigned 64-bit integer (which loads as `decimal(20,0)`, so that type keeps SQLLoad's reader) as a signed one, and JSON or a native type its own way. +pub(in crate::adapters) fn direct_problem(spec: &TableSpec, read: &FilesRead) -> Option { + let files = spec.files.as_ref()?; + let StoreKind::Gcs { .. } = &files.location.store else { + return None; + }; + let problems = [ + (files.encoding.format != SourceFormat::Parquet) + .then(|| "only Parquet files are read by BigQuery itself".to_owned()), + custom_endpoint(&files.location.store, &environment_variable), + read.columns.iter().any(|column| column.name == SOURCE_ROW_COLUMN).then(|| { + format!( + "BigQuery doesn't report a row's position in its file, so it can't fill {SOURCE_ROW_COLUMN}; to have BigQuery load the files itself, add {SOURCE_ROW_COLUMN} to exclude_columns" + ) + }), + (!spec.masked.columns.is_empty()).then(|| "the table masks columns".to_owned()), + files_direct_problem(files, read), + unread_name( + files + .table + .columns + .iter() + .map(|column| column.name.as_str()) + .filter(|name| !own_column(files, name)), + ), + read.columns.iter().find_map(|column| { + let held = match &column.column_type { + ColumnType::Uuid => "a UUID, which BigQuery reads from Parquet as bytes", + ColumnType::Array(_) => "a list, which BigQuery reads from Parquet as a record", + ColumnType::Json => "JSON, which BigQuery reads from Parquet as text", + ColumnType::Native(_) => "a native type, which SQLLoad converts itself", + ColumnType::Decimal { + precision: 20, + scale: 0, + } => "decimal(20,0), the type of an unsigned 64-bit integer, which BigQuery reads from Parquet as a signed one and fails on values above 9223372036854775807", + _ => return None, + }; + Some(format!("{} is {held}", column.name)) + }), + ]; + problems.into_iter().flatten().next() +} + +/// Whether `name` is a value SQLLoad gives each file (its name, version, row number or a partition), which no file holds. +fn own_column(files: &FilesSource, name: &str) -> bool { + [SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN, SOURCE_ROW_COLUMN].contains(&name) + || files + .partition_types + .iter() + .any(|(partition, _)| partition == name) +} + +/// `name`'s value in the process environment; unset when it isn't Unicode, as SQLLoad's reader takes it. +fn environment_variable(name: &str) -> Option { + match std::env::var(name) { + Ok(value) => Some(value), + Err(std::env::VarError::NotPresent | std::env::VarError::NotUnicode(_)) => None, + } +} diff --git a/crates/sqlload/src/adapters/bigquery/main/plan/mod.rs b/crates/sqlload/src/adapters/bigquery/main/plan/mod.rs new file mode 100644 index 000000000..7f76ebb2d --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/main/plan/mod.rs @@ -0,0 +1,2 @@ +pub(in crate::adapters) mod _check_files; +pub(in crate::adapters) mod _direct_problem; diff --git a/crates/sqlload/src/adapters/bigquery/tests.rs b/crates/sqlload/src/adapters/bigquery/tests.rs index 5e7057803..4852a7487 100644 --- a/crates/sqlload/src/adapters/bigquery/tests.rs +++ b/crates/sqlload/src/adapters/bigquery/tests.rs @@ -4,5 +4,7 @@ mod arrow_rows; mod dialect; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_pull_query.rs"] +mod pull_query; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlload/src/adapters/bigquery/tests/test_pull_query.rs b/crates/sqlload/src/adapters/bigquery/tests/test_pull_query.rs new file mode 100644 index 000000000..70e34c563 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/tests/test_pull_query.rs @@ -0,0 +1,146 @@ +use crate::adapters::bigquery::_helpers::dialect::pull::{ + PulledFile, PulledGroup, pull_queries, pull_query, +}; +use crate::adapters::bigquery::constants::{MAX_FILES_PER_QUERY, MAX_QUERY_BYTES}; +use crate::adapters::bigquery::tests::helpers::{desired, orders}; +use crate::adapters::bigquery::tests::test_types::PullQueriesTestCase; +use crate::adapters::bigquery::tests::test_types::PullQueryTestCase; +use crate::column_types::models::ColumnType; +use crate::load_naming::models::Compat; + +#[test] +fn given_parquet_files_in_gcs_when_bigquery_reads_them_itself_then_each_column_comes_from_the_right_place() + { + let file = |url: &str, day: Option<&str>| PulledFile { + url: url.to_owned(), + constants: vec![ + ("day".to_owned(), day.map(str::to_owned)), + ("_sqlload_source_file".to_owned(), Some(url.to_owned())), + ], + }; + let test_cases = [PullQueryTestCase { + description: "each group of files holding the same columns is its own external table: its columns are cast from their names in the files, a partition and the file's name come from the row's _FILE_NAME (NULL where its folder lacks it, and an error for a file not in the batch, never a NULL), the load id is the batch's, and a column the group's files lack is a typed NULL; types are cast to without parameters, which a cast can't name", + columns: vec![ + desired("id", ColumnType::Int), + desired( + "amount", + ColumnType::Decimal { + precision: 12, + scale: 2, + }, + ), + desired("code", ColumnType::Varchar(8)), + desired("day", ColumnType::Date), + desired("_sqlload_source_file", ColumnType::Text), + desired("_sqlload_load_id", ColumnType::Text), + desired("note", ColumnType::Text), + ], + groups: vec![ + PulledGroup { + files: vec![ + file("gs://b/x/a.parquet", Some("2026-01-01")), + file("gs://b/x/it's.parquet", None), + ], + in_files: vec![ + ("id".to_owned(), "Order_ID".to_owned()), + ("amount".to_owned(), "amount".to_owned()), + ], + }, + PulledGroup { + files: vec![file("gs://b/x/c.parquet", Some("2026-01-02"))], + in_files: vec![("id".to_owned(), "id".to_owned())], + }, + ], + expected_query: "INSERT INTO `p`.`raw`.`orders` (`id`, `amount`, `code`, `day`, `_sqlload_source_file`, `_sqlload_load_id`, `note`) SELECT CAST(f.`Order_ID` AS INT64), CAST(f.`amount` AS NUMERIC), CAST(NULL AS STRING), CAST(CASE _FILE_NAME WHEN 'gs://b/x/a.parquet' THEN '2026-01-01' WHEN 'gs://b/x/it\\'s.parquet' THEN CAST(NULL AS STRING) ELSE ERROR(CONCAT('sqlload: unexpected file ', _FILE_NAME)) END AS DATE), CAST(CASE _FILE_NAME WHEN 'gs://b/x/a.parquet' THEN 'gs://b/x/a.parquet' WHEN 'gs://b/x/it\\'s.parquet' THEN 'gs://b/x/it\\'s.parquet' ELSE ERROR(CONCAT('sqlload: unexpected file ', _FILE_NAME)) END AS STRING), CAST('L1' AS STRING), CAST(NULL AS STRING) FROM sqlload_files_0 AS f UNION ALL SELECT CAST(f.`id` AS INT64), CAST(NULL AS NUMERIC), CAST(NULL AS STRING), CAST(CASE _FILE_NAME WHEN 'gs://b/x/c.parquet' THEN '2026-01-02' ELSE ERROR(CONCAT('sqlload: unexpected file ', _FILE_NAME)) END AS DATE), CAST(CASE _FILE_NAME WHEN 'gs://b/x/c.parquet' THEN 'gs://b/x/c.parquet' ELSE ERROR(CONCAT('sqlload: unexpected file ', _FILE_NAME)) END AS STRING), CAST('L1' AS STRING), CAST(NULL AS STRING) FROM sqlload_files_1 AS f", + expected_definitions: r#"{"sqlload_files_0":{"decimalTargetTypes":["NUMERIC","BIGNUMERIC","STRING"],"sourceFormat":"PARQUET","sourceUris":["gs://b/x/a.parquet","gs://b/x/it's.parquet"]},"sqlload_files_1":{"decimalTargetTypes":["NUMERIC","BIGNUMERIC","STRING"],"sourceFormat":"PARQUET","sourceUris":["gs://b/x/c.parquet"]}}"#, + }]; + for test_case in test_cases { + let (query, definitions) = pull_query( + &orders(), + (&test_case.columns, false), + &test_case.groups, + (Compat::Sqlload, "L1", "2026-11-06T00:00:00Z"), + ); + assert_eq!(query, test_case.expected_query, "{}", test_case.description); + assert_eq!( + definitions.to_string(), + test_case.expected_definitions, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_batch_too_big_for_one_query_when_bigquery_reads_it_then_it_is_split_into_queries_under_the_limits() + { + let group = |files: usize, url_bytes: usize, in_files: &str| PulledGroup { + files: (0..files) + .map(|index| PulledFile { + url: format!("gs://b/{index:0>url_bytes$}.parquet"), + constants: vec![ + ("day".to_owned(), Some(format!("{index:0>url_bytes$}"))), + ("_sqlload_source_file".to_owned(), Some(format!("{index}"))), + ], + }) + .collect(), + in_files: vec![("id".to_owned(), in_files.to_owned())], + }; + let test_cases = [ + PullQueriesTestCase { + description: "a small batch is one query", + groups: vec![group(3, 10, "id"), group(2, 10, "ID2")], + expected_queries: 1, + }, + PullQueriesTestCase { + description: "long keys over two schema groups split by size, each file in exactly one query", + groups: vec![group(1000, 500, "id"), group(1000, 500, "id2")], + expected_queries: 4, + }, + PullQueriesTestCase { + description: "more files than one query takes split by count, even when short", + groups: vec![group(1500, 1, "id")], + expected_queries: 2, + }, + ]; + let columns = [ + desired("id", ColumnType::Int), + desired("day", ColumnType::Text), + desired("_sqlload_source_file", ColumnType::Text), + ]; + for test_case in test_cases { + let queries = pull_queries( + &orders(), + (&columns, false), + &test_case.groups, + (Compat::Sqlload, "L1", "2026-11-06T00:00:00Z"), + ); + let files: usize = test_case.groups.iter().map(|group| group.files.len()).sum(); + let urls: usize = queries + .iter() + .map(|(_, definitions)| definitions.to_string().matches("gs://b/").count()) + .sum(); + let largest = queries.iter().map(|(query, _)| query.len()).max(); + let most_files = queries + .iter() + .map(|(_, definitions)| definitions.to_string().matches("gs://b/").count()) + .max(); + assert_eq!( + queries.len(), + test_case.expected_queries, + "{}", + test_case.description + ); + assert_eq!(urls, files, "{}", test_case.description); + assert!( + largest <= Some(MAX_QUERY_BYTES), + "{}", + test_case.description + ); + assert!( + most_files <= Some(MAX_FILES_PER_QUERY), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/bigquery/tests/test_types.rs b/crates/sqlload/src/adapters/bigquery/tests/test_types.rs index 72be58e68..5668ba7bb 100644 --- a/crates/sqlload/src/adapters/bigquery/tests/test_types.rs +++ b/crates/sqlload/src/adapters/bigquery/tests/test_types.rs @@ -29,3 +29,17 @@ pub(crate) struct AppendRequestsTestCase { /// The ids the requests decode back to, in order, as `first..=last` and their count. pub(crate) expected_ids: String, } + +pub(crate) struct PullQueryTestCase { + pub(crate) description: &'static str, + pub(crate) columns: Vec, + pub(crate) groups: Vec, + pub(crate) expected_query: &'static str, + pub(crate) expected_definitions: &'static str, +} + +pub(crate) struct PullQueriesTestCase { + pub(crate) description: &'static str, + pub(crate) groups: Vec, + pub(crate) expected_queries: usize, +} diff --git a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs index 8bf88c5c7..c2ae319c5 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs @@ -1,5 +1,6 @@ //! Whether a destination reaches the files it would read itself. +use crate::adapters::bigquery::main::plan::_check_files::check_files as bigquery_check_files; use crate::adapters::databricks::main::stage::_check_folder::check_folder; use crate::adapters::databricks::main::stage::_external_locations::external_locations; use crate::adapters::databricks::models::DatabricksDestination; @@ -11,7 +12,7 @@ use crate::loader::models::FilesSource; use crate::routing::models::Store; use crate::store_paths::models::StoreKind; -/// Check, at plan, that `destination` reads the files of `source` in S3 or GCS where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers, and DuckDB reads it from GCS (Databricks reads only S3). An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix. +/// Check, at plan, that `destination` reads the files of `source` in S3 or GCS where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers, DuckDB reads it from GCS, and BigQuery reads its schema from GCS in a dry run (Databricks reads only S3). An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix. pub async fn check_reach( destination: &Destination, (source, unload_to): (Option<&FilesSource>, &[Store]), @@ -40,6 +41,20 @@ pub async fn check_reach( { databricks_reach(databricks, first.map(|_| folder.as_str())).await } + Destination::BigQuery(bigquery) + if matches!(source.location.store, StoreKind::Gcs { .. }) => + { + let Some(data) = first else { + return Ok(()); + }; + let url = format!("{folder}{}", data.file.relative_key); + bigquery_check_files(bigquery, &url).await.map_err(|error| { + format!( + "{url} cannot be read by BigQuery: {}. Fix: BigQuery reads the files with its own connection's service account, not the gcs connection's key: give that service account roles/storage.objectViewer on the bucket, and let no VPC Service Controls perimeter keep BigQuery out", + first_line(&error.to_string()) + ) + }) + } #[cfg(feature = "duckdb")] Destination::DuckDb(duckdb) => { let Some(data) = first else { diff --git a/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs b/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs index f2bc3fc70..1d46e643b 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/direct_blocked.rs @@ -1,6 +1,7 @@ //! Why the destination cannot load a files table's files exactly by reading them itself. use crate::adapters::bigquery::constants::NO_INTERNAL_STAGE_COPY as BIGQUERY_NO_INTERNAL_STAGE_COPY; +use crate::adapters::bigquery::main::plan::_direct_problem::direct_problem as bigquery_direct_problem; use crate::adapters::clickhouse::constants::NO_INTERNAL_STAGE_COPY as CLICKHOUSE_NO_INTERNAL_STAGE_COPY; use crate::adapters::clickhouse::main::plan::_direct_problem::direct_problem as clickhouse_direct_problem; use crate::adapters::databricks::constants::NO_INTERNAL_STAGE_COPY; @@ -15,7 +16,7 @@ use crate::column_types::models::Warehouse; use crate::files_source::models::FilesRead; use crate::loader::models::TableSpec; -/// Why the destination cannot load the planned `read` of a files table exactly from its files: for DuckDB, by reading them itself wherever they are; for ClickHouse, by reading them from S3; for Databricks, by `COPY`ing them from S3 through an external location; for Snowflake, under the external `stage` that covers them, or copied unchanged to an internal stage (`copy_to_internal_stage`); `None` when it can, or when neither applies. +/// Why the destination cannot load the planned `read` of a files table exactly from its files: for DuckDB, by reading them itself wherever they are; for ClickHouse, by reading them from S3; for Databricks, by `COPY`ing them from S3 through an external location; for BigQuery, by querying them in GCS; for Snowflake, under the external `stage` that covers them, or copied unchanged to an internal stage (`copy_to_internal_stage`); `None` when it can, or when neither applies. pub fn direct_blocked( spec: &TableSpec, read: &FilesRead, @@ -34,6 +35,7 @@ pub fn direct_blocked( (true, _) => Some(refused.to_owned()), (false, Warehouse::ClickHouse) => clickhouse_direct_problem(spec, read), (false, Warehouse::Databricks) => databricks_direct_problem(spec, read), + (false, Warehouse::BigQuery) => bigquery_direct_problem(spec, read), (false, _) => None, }; } diff --git a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs index 3f42d42f0..aec662e55 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs @@ -7,7 +7,7 @@ use crate::adapters::snowflake::models::StageRoute; use crate::routing::models::WriterAccepts; use crate::routing::models::{FileFormat, Store}; -/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself, MotherDuck Parquet in S3 or GCS on its servers, and ClickHouse Parquet in S3), or what its stage route takes. +/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself, MotherDuck Parquet in S3 or GCS on its servers, and ClickHouse Parquet in S3, BigQuery Parquet in GCS), or what its stage route takes. pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> WriterAccepts { match destination { #[cfg(feature = "duckdb")] @@ -37,12 +37,16 @@ pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> pull_formats: vec![FileFormat::Parquet], ..WriterAccepts::default() }, - Destination::Files(_) | Destination::BigQuery(_) | Destination::Postgres(_) => { - WriterAccepts { - arrow: true, - ..WriterAccepts::default() - } - } + Destination::BigQuery(_) => WriterAccepts { + arrow: true, + pulls_from: vec![Store::ObjectStore("gs://".to_owned())], + pull_formats: vec![FileFormat::Parquet], + ..WriterAccepts::default() + }, + Destination::Files(_) | Destination::Postgres(_) => WriterAccepts { + arrow: true, + ..WriterAccepts::default() + }, Destination::Snowflake(_) => snowflake_accepts(stage), Destination::Databricks(databricks) => databricks_accepts(databricks), } diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs index fc200ae63..fcbba974d 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_files.rs @@ -4,7 +4,7 @@ use crate::adapters::contract::models::{DirectBatch, DirectLoaded}; use crate::adapters::dispatch::models::Writer; use crate::loader::errors::LoaderError; -/// Load `batch` by having the destination read the source's files itself (Snowflake's `COPY` from an external stage, ClickHouse's `s3()`, Databricks' `COPY INTO` from S3); `None` when SQLLoad must read the batch. +/// Load `batch` by having the destination read the source's files itself (Snowflake's `COPY` from an external stage, ClickHouse's `s3()`, Databricks' `COPY INTO` from S3, BigQuery's query of Parquet in GCS); `None` when SQLLoad must read the batch. pub async fn load_files( writer: &mut Writer, batch: &DirectBatch<'_>, @@ -13,6 +13,7 @@ pub async fn load_files( Writer::Snowflake(snowflake) => snowflake.copy_files(batch).await, Writer::ClickHouse(clickhouse) => clickhouse.copy_files(batch).await, Writer::Databricks(databricks) => databricks.copy_files(batch).await, + Writer::BigQuery(bigquery) => bigquery.copy_files(batch).await, #[cfg(feature = "duckdb")] Writer::DuckDb(duckdb) => duckdb.copy_files(batch).await, _ => Ok(None), diff --git a/crates/sqlload/src/bigquery_api/main/load_job.rs b/crates/sqlload/src/bigquery_api/main/load_job.rs index d4471155c..dfac68dea 100644 --- a/crates/sqlload/src/bigquery_api/main/load_job.rs +++ b/crates/sqlload/src/bigquery_api/main/load_job.rs @@ -7,22 +7,25 @@ use crate::bigquery_api::constants::{CONFLICT, JOB_POLL_FIRST, JOB_POLL_MAX}; use crate::bigquery_api::models::BigQueryClient; use crate::http::errors::ApiError; -/// Run the load job `job` (a `jobs.insert` body naming its own `jobId`) in the client's project and wait until it is done; returns its final resource, whose `status.errorResult` says whether it loaded. With `file`, the file is sent along with the job in a resumable upload, so no bucket is needed; without, the job reads the `sourceUris` it names. A job id already taken is not run again: that job is waited on instead. +/// Run the load job `job` (a `jobs.insert` body naming its own `jobId`) in the client's project and wait until it is done; returns its final resource, whose `status.errorResult` says whether it loaded. With `file`, the file is sent along with the job in a resumable upload, so no bucket is needed; without, the job reads the `sourceUris` or runs the query it names, and its start is retried on a transport failure, a 429 or a 5xx. A job id already taken (as by a retried start that reached BigQuery) is not run again: that job is waited on instead. pub async fn load_job( client: &BigQueryClient, job: &Value, file: Option, ) -> Result { let job_id = job["jobReference"]["jobId"].as_str().unwrap_or_default(); - let started = match file { + let started: Result = match file { None => { let url = format!("{}/projects/{}/jobs", client.base_url, client.project); - send(client, |http| http.post(&url).json(job)).await + json_with_retries(client, |http| http.post(&url).json(job)).await } - Some(file) => upload(client, job, file).await, + Some(file) => match upload(client, job, file).await { + Ok(response) => response.json().await.map_err(ApiError::from), + Err(error) => Err(error), + }, }; let mut resource: Value = match started { - Ok(response) => response.json().await?, + Ok(resource) => resource, Err(error) if error.status() == Some(CONFLICT) => job_resource(client, job_id).await?, Err(error) => return Err(error), }; diff --git a/crates/sqlload/src/bigquery_api/main/query_schema.rs b/crates/sqlload/src/bigquery_api/main/query_schema.rs index b38b25199..561210656 100644 --- a/crates/sqlload/src/bigquery_api/main/query_schema.rs +++ b/crates/sqlload/src/bigquery_api/main/query_schema.rs @@ -7,15 +7,20 @@ use crate::bigquery_api::_helpers::requests::json_with_retries; use crate::bigquery_api::models::BigQueryClient; use crate::http::errors::ApiError; -/// The schema fields (`name`, `type`, `mode` and sizes, as `tables.get` gives them) of `statement`'s result, from a dry run: nothing is read or billed. +/// The schema fields (`name`, `type`, `mode` and sizes, as `tables.get` gives them) of `statement`'s result, from a dry run: nothing is read or billed. `definitions` are the query's temporary external tables (`tableDefinitions`), whose files' schemas the dry run reads, failing where BigQuery can't. pub async fn query_schema( client: &BigQueryClient, statement: &str, + definitions: Option<&Value>, ) -> Result, ApiError> { + let mut query = json!({ "query": statement, "useLegacySql": false }); + if let Some(definitions) = definitions { + query["tableDefinitions"] = definitions.clone(); + } let body = json!({ "configuration": { "dryRun": true, - "query": { "query": statement, "useLegacySql": false }, + "query": query, }, "jobReference": { "projectId": client.project, "location": client.location }, }); diff --git a/crates/sqlload/src/bigquery_source/main/dry_count.rs b/crates/sqlload/src/bigquery_source/main/dry_count.rs index 3309b9856..6b0e1165e 100644 --- a/crates/sqlload/src/bigquery_source/main/dry_count.rs +++ b/crates/sqlload/src/bigquery_source/main/dry_count.rs @@ -13,6 +13,6 @@ pub async fn dry_count( filter: &str, ) -> Result<(), BigQuerySourceError> { let query = count_query(&reader.project, (table, sql), filter); - query_schema(&reader.client, &query).await?; + query_schema(&reader.client, &query, None).await?; Ok(()) } diff --git a/crates/sqlload/src/bigquery_source/main/read_catalog.rs b/crates/sqlload/src/bigquery_source/main/read_catalog.rs index 9e6ec8515..89fe3a091 100644 --- a/crates/sqlload/src/bigquery_source/main/read_catalog.rs +++ b/crates/sqlload/src/bigquery_source/main/read_catalog.rs @@ -19,11 +19,13 @@ pub async fn read_catalog( false => catalog_tables(reader, (include, exclude)).await?, }; for (name, sql) in queries { - let fields = query_schema(&reader.client, &format!("SELECT * FROM {}", subquery(sql))) - .await - .map_err(|error| { - BigQuerySourceError::Invalid(format!("query table {name}: {error}")) - })?; + let fields = query_schema( + &reader.client, + &format!("SELECT * FROM {}", subquery(sql)), + None, + ) + .await + .map_err(|error| BigQuerySourceError::Invalid(format!("query table {name}: {error}")))?; tables.push(SourceTable { split_pages: None, schema: String::new(), diff --git a/crates/sqlload/src/loader/tests/helpers.rs b/crates/sqlload/src/loader/tests/helpers.rs index 2d2e0c94e..44003fe9d 100644 --- a/crates/sqlload/src/loader/tests/helpers.rs +++ b/crates/sqlload/src/loader/tests/helpers.rs @@ -20,9 +20,9 @@ use crate::loader::_helpers::table_plan::reader_offer; use crate::loader::errors::LoaderError; use crate::loader::models::{FilesSource, PlanDetail, PlanItem, TableSpec, TableTransform}; use crate::loader::tests::test_types::{ - ChildPathsTestCase, DeclaredProblemsTestCase, GcsReaderTestCase, InferredTableTestCase, - ItemSpec, OlderFilesTestCase, ReloadOrderTestCase, SourceSpellingTestCase, - TransformReaderTestCase, + BigQueryReaderTestCase, ChildPathsTestCase, DeclaredProblemsTestCase, GcsReaderTestCase, + InferredTableTestCase, ItemSpec, OlderFilesTestCase, ReloadOrderTestCase, + SourceSpellingTestCase, TransformReaderTestCase, }; use crate::postgres_source::models::{PgType, SourceColumn, SourceTable}; use crate::project::main::parse_python_project::parse_python_project; @@ -579,6 +579,53 @@ pub(crate) fn gcs_reader(test_case: &GcsReaderTestCase) -> Result Ok(reader_offer(&spec, Some(&read), None).direct_blocked) } +/// Why BigQuery can't load a Parquet files table's files in GCS itself, writing the case's columns, with the case's endpoint on the connection. +pub(crate) fn bigquery_reader( + test_case: &BigQueryReaderTestCase, +) -> Result, String> { + let mut spec = files_spec(StoreKind::Gcs { + bucket: "lake".to_owned(), + service_account_key: None, + endpoint: test_case.endpoint.map(str::to_owned), + hmac: None, + })?; + spec.warehouse = crate::column_types::models::Warehouse::BigQuery; + spec.files.as_mut().ok_or("no files source")?.table.columns = test_case + .columns + .iter() + .filter(|(name, _)| !name.starts_with("_sqlload_")) + .map( + |(name, column_type)| crate::files_source::models::FileColumn { + name: (*name).to_owned(), + column_type: Ok(column_type.clone()), + direct_refusal: None, + stored_read: crate::files_source::models::StoredRead::Plain, + lists: None, + repeat_of: None, + }, + ) + .collect(); + let read = FilesRead { + files: Vec::new(), + encoding: FilesEncoding::default(), + checks: TextChecks::default(), + columns: test_case + .columns + .iter() + .map( + |(name, column_type)| crate::schema_plan::models::DesiredColumn { + name: (*name).to_owned(), + column_type: column_type.clone(), + origin: crate::schema_plan::models::ColumnOrigin::Source, + }, + ) + .collect(), + nanos: Vec::new(), + batch_rows: 1000, + }; + Ok(reader_offer(&spec, Some(&read), None).direct_blocked) +} + /// A Parquet files table in `drop/` of `store`, loading into DuckDB. fn files_spec(store: StoreKind) -> Result { let mut spec = python_spec("{}")?; diff --git a/crates/sqlload/src/loader/tests/test_transform_reader.rs b/crates/sqlload/src/loader/tests/test_transform_reader.rs index fe29989e2..a70a61912 100644 --- a/crates/sqlload/src/loader/tests/test_transform_reader.rs +++ b/crates/sqlload/src/loader/tests/test_transform_reader.rs @@ -1,5 +1,8 @@ -use crate::loader::tests::helpers::{gcs_reader, transform_reader}; -use crate::loader::tests::test_types::{GcsReaderTestCase, TransformReaderTestCase}; +use crate::column_types::models::ColumnType; +use crate::loader::tests::helpers::{bigquery_reader, gcs_reader, transform_reader}; +use crate::loader::tests::test_types::{ + BigQueryReaderTestCase, GcsReaderTestCase, TransformReaderTestCase, +}; #[test] fn given_a_files_table_with_a_transform_or_a_parse_when_routing_then_sqlload_reads_the_files_itself() @@ -70,3 +73,102 @@ fn given_parquet_files_in_gcs_when_routing_into_duckdb_then_duckdb_reads_them_on } Ok(()) } + +#[test] +fn given_parquet_files_in_gcs_when_routing_into_bigquery_then_bigquery_reads_them_only_without_a_row_number() +-> Result<(), String> { + let test_cases = [ + BigQueryReaderTestCase { + description: "a table writing _sqlload_source_row keeps SQLLoad's reader, as BigQuery can't number a file's rows, and the reason names the opt-in", + columns: vec![ + ("id", ColumnType::Int), + ("_sqlload_source_row", ColumnType::Int), + ], + endpoint: None, + expected_reason: Some( + "BigQuery doesn't report a row's position in its file, so it can't fill _sqlload_source_row; to have BigQuery load the files itself, add _sqlload_source_row to exclude_columns", + ), + }, + BigQueryReaderTestCase { + description: "without it BigQuery reads the files itself", + columns: vec![ + ("id", ColumnType::Int), + ( + "amount", + ColumnType::Decimal { + precision: 12, + scale: 2, + }, + ), + ("at", ColumnType::TimestampTz), + ], + endpoint: None, + expected_reason: None, + }, + BigQueryReaderTestCase { + description: "a UUID, which BigQuery reads from Parquet as bytes, keeps SQLLoad's reader", + columns: vec![("id", ColumnType::Uuid)], + endpoint: None, + expected_reason: Some("id is a UUID, which BigQuery reads from Parquet as bytes"), + }, + BigQueryReaderTestCase { + description: "decimal(20,0), the type an unsigned 64-bit integer loads as, keeps SQLLoad's reader, as BigQuery reads it as a signed one", + columns: vec![( + "n", + ColumnType::Decimal { + precision: 20, + scale: 0, + }, + )], + endpoint: None, + expected_reason: Some( + "n is decimal(20,0), the type of an unsigned 64-bit integer, which BigQuery reads from Parquet as a signed one and fails on values above 9223372036854775807", + ), + }, + BigQueryReaderTestCase { + description: "a list keeps SQLLoad's reader", + columns: vec![("tags", ColumnType::Array(Box::new(ColumnType::Text)))], + endpoint: None, + expected_reason: Some("tags is a list, which BigQuery reads from Parquet as a record"), + }, + BigQueryReaderTestCase { + description: "a column name BigQuery renames keeps SQLLoad's reader, as BigQuery would read it under another name", + columns: vec![("Order ID", ColumnType::Int)], + endpoint: None, + expected_reason: Some( + "BigQuery would not read column \"Order ID\" as it is: BigQuery renames a name that isn't letters, digits and underscores starting with a letter or underscore", + ), + }, + BigQueryReaderTestCase { + description: "two names differing only in case keep SQLLoad's reader", + columns: vec![("id", ColumnType::Int), ("ID", ColumnType::Text)], + endpoint: None, + expected_reason: Some( + "BigQuery would not read column \"ID\" as it is: another column has the same name in another case, and BigQuery's names ignore case", + ), + }, + BigQueryReaderTestCase { + description: "a name with a prefix BigQuery reserves keeps SQLLoad's reader", + columns: vec![("_file_name", ColumnType::Text)], + endpoint: None, + expected_reason: Some( + "BigQuery would not read column \"_file_name\" as it is: _FILE_ is a prefix BigQuery reserves", + ), + }, + BigQueryReaderTestCase { + description: "an emulator's endpoint is SQLLoad's to follow, not BigQuery's", + columns: vec![("id", ColumnType::Int)], + endpoint: Some("http://127.0.0.1:4443"), + expected_reason: Some("the GCS connection sets a custom endpoint"), + }, + ]; + for test_case in test_cases { + assert_eq!( + bigquery_reader(&test_case)?.as_deref(), + test_case.expected_reason, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs index 1d930bf5b..837dacc37 100644 --- a/crates/sqlload/src/loader/tests/test_types.rs +++ b/crates/sqlload/src/loader/tests/test_types.rs @@ -274,3 +274,14 @@ pub(crate) struct GcsReaderTestCase { /// Why SQLLoad must read the files instead of DuckDB, if it must. pub(crate) expected_reason: Option<&'static str>, } + +/// A Parquet files table in GCS loading into BigQuery. +pub(crate) struct BigQueryReaderTestCase { + pub(crate) description: &'static str, + /// The columns the load writes, with their types. + pub(crate) columns: Vec<(&'static str, crate::column_types::models::ColumnType)>, + /// The gcs connection's custom endpoint, if any. + pub(crate) endpoint: Option<&'static str>, + /// Why SQLLoad must read the files instead of BigQuery, if it must. + pub(crate) expected_reason: Option<&'static str>, +} diff --git a/website/public/index.html b/website/public/index.html index 3a2778a52..91b1d7198 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -173,7 +173,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 or Google Cloud Storage into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, Parquet in S3 or Google Cloud Storage into MotherDuck on its servers, and Parquet in Google Cloud Storage into BigQuery as a temporary external table. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/concepts/projects.mdx b/website/src/content/docs/docs/concepts/projects.mdx
index 0b2e03a7b..efadba865 100644
--- a/website/src/content/docs/docs/concepts/projects.mdx
+++ b/website/src/content/docs/docs/concepts/projects.mdx
@@ -129,7 +129,7 @@ Every loaded table gets columns that say where each row came from:
 | `_sqlload_loaded_at` | every table | When that load ran. |
 | `_sqlload_source_file` | files sources | The file the row came from. |
 | `_sqlload_source_version` | files sources | The file's version when it was loaded. |
-| `_sqlload_source_row` | files sources | The row's number in its file, from 1. |
+| `_sqlload_source_row` | files sources | The row's number in its file, from 1. BigQuery doesn't report a row's position in its file, so a BigQuery table that writes it has SQLLoad read the files; [exclude it](/docs/targets/bigquery/#parquet-in-google-cloud-storage) to have BigQuery load them itself. |
 | `_sqlload_source_row` | Google Sheets tables | The row's number in the sheet, as Google Sheets numbers it: a range starting at `A3` starts at 3, and a skipped empty row leaves a gap. Not on a table with a `transform`. |
 
 `plan` lists them as `+ _sqlload_load_id, _loaded_at`. Files sources use `_sqlload_source_file`
diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx
index 9d5bb98f1..62104e77a 100644
--- a/website/src/content/docs/docs/configuration.mdx
+++ b/website/src/content/docs/docs/configuration.mdx
@@ -173,6 +173,10 @@ a bucket's Parquet files themselves, create an HMAC key for a service account th
 (Cloud Storage → Settings → Interoperability) and set `hmac_key_id` and `hmac_secret`. Without one,
 SQLLoad reads the files and sends their rows, and `sqlload plan -v ` says why.
 
+BigQuery needs no HMAC key: it reads a bucket's Parquet files with its own connection's service
+account, on a table that excludes `_sqlload_source_row`; see [Parquet in Google Cloud
+Storage](/docs/targets/bigquery/#parquet-in-google-cloud-storage).
+
 ### `filesystem`
 
 | Key | Meaning |
diff --git a/website/src/content/docs/docs/guides/files.mdx b/website/src/content/docs/docs/guides/files.mdx
index cdc27a594..7d1227b2b 100644
--- a/website/src/content/docs/docs/guides/files.mdx
+++ b/website/src/content/docs/docs/guides/files.mdx
@@ -143,7 +143,8 @@ hmac_secret = "${ENV:GCS_HMAC_SECRET}"
 ```
 
 Without `service_account_key`, SQLLoad uses Google's application default credentials. The HMAC
-key is optional: DuckDB and MotherDuck read Parquet in GCS themselves only with one. See the
+key is optional: DuckDB and MotherDuck read Parquet in GCS themselves only with one; BigQuery reads
+it with its own service account. See the
 [`gcs` connection](/docs/configuration/#gcs).
 
 ## Load files from an SFTP server
@@ -189,6 +190,7 @@ SQLLoad. `plan` names the route on each table's `route` line:
 | DuckDB | Parquet in S3, GCS (with an HMAC key) or a local folder, with `read_parquet`. |
 | Snowflake | Parquet, JSON, NDJSON and CSV under the `url` of an `external_stage` load method, with `COPY`. |
 | ClickHouse | Parquet in S3, with `s3()`. |
+| BigQuery | Parquet in GCS, as temporary external tables, on a table that excludes `_sqlload_source_row`. |
 
 SQLLoad reads the files itself for a table that changes its rows before they load: one with a
 `transform`, a `parse`, `checks`, `row_id`, `unnest`, a `flatten` of the files' objects, or a column
diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx
index a4661a133..430463293 100644
--- a/website/src/content/docs/docs/sources/files.mdx
+++ b/website/src/content/docs/docs/sources/files.mdx
@@ -739,7 +739,10 @@ records, unless you pin them with `columns`. `infer_types = false` loads every
 column not pinned as `text`, CSV fields exactly as written; see [Loading every column as
 text](/docs/concepts/types/#loading-every-column-as-text). Each row also gets `_sqlload_source_file`,
 `_sqlload_source_version` and `_sqlload_source_row` ([Columns SQLLoad
-adds](/docs/concepts/projects/#columns-sqlload-adds)).
+adds](/docs/concepts/projects/#columns-sqlload-adds)). BigQuery doesn't report a row's position in
+its file: into BigQuery, Parquet in GCS loads through BigQuery itself only when `exclude_columns`
+drops `_sqlload_source_row`; see [Parquet in Google Cloud
+Storage](/docs/targets/bigquery/#parquet-in-google-cloud-storage).
 
 Parquet list, struct and map columns load as `json`.
 A list becomes an array, a struct becomes an object with its fields in order, and a map becomes an
@@ -884,6 +887,8 @@ SQLLoad:
 - **Snowflake** runs `COPY` from an external stage over the files' bucket, for Parquet, JSON,
   NDJSON and CSV.
 - **ClickHouse** reads Parquet in S3 with `s3()`.
+- **BigQuery** queries Parquet in GCS as temporary external tables, on a table that excludes
+  `_sqlload_source_row`, which BigQuery can't fill.
 
 `plan` names the route for each table. See
 [Let the warehouse read the files](/docs/guides/files/#let-the-warehouse-read-the-files) and
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index ae2860dd2..8bef8fc78 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -16,7 +16,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 | DuckDB | Tables and views of a database file, by `include` patterns, and query results | One `SELECT` on a read-only connection, streamed as Arrow. See [DuckDB](/docs/sources/duckdb/). |
 | MotherDuck (beta) | Tables and views of one database, by `include` patterns, and query results | One `SELECT` run by a separate DuckDB 1.5 process (`sqlload[motherduck]`), its Arrow batches handed over one at a time. See [MotherDuck](/docs/sources/motherduck/). |
 | Files in S3 | Parquet, CSV, JSON, NDJSON, Excel (xlsx) and XML; gzip or zstd for text formats; ZIP archives of any of them; any file name through `pattern`. See [Files](/docs/sources/files/#excel-xml-and-zip-files). | The destination itself where it can load them exactly, otherwise SQLLoad's reader |
-| Files in Google Cloud Storage | As for S3 | As for S3; DuckDB and MotherDuck read Parquet themselves with an HMAC key. See [`gcs`](/docs/configuration/#gcs). |
+| Files in Google Cloud Storage | As for S3 | As for S3; DuckDB and MotherDuck read Parquet themselves with an HMAC key, and BigQuery with its own service account. See [`gcs`](/docs/configuration/#gcs). |
 | Files in a local folder | As for S3 | As for S3 |
 | Files on an SFTP server | As for S3 | SQLLoad's reader, over SFTP. See [Files](/docs/sources/files/#files-on-an-sftp-server). |
 | Google Sheets | Tabs or A1 ranges, values typed by their cells | The Sheets API with a service-account key, into Arrow. See [Google Sheets](/docs/sources/google-sheets/). |
@@ -29,7 +29,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 | Destination | Write strategies | How data arrives |
 |-------------|------------------|------------------|
 | Snowflake | `append`, `merge`, `delete_insert`, `replace` | Parquet staged on an external, user, table or named stage, then `COPY`; or Snowpipe Streaming, or `INSERT`, by `load_method`. Files under an external stage load with `COPY` directly. |
-| BigQuery (beta) | `append`, `merge`, `delete_insert`, `replace` | Arrow streamed through the Storage Write API into a staging table, then one query job per write. See [BigQuery](/docs/targets/bigquery/). |
+| BigQuery (beta) | `append`, `merge`, `delete_insert`, `replace` | Arrow streamed through the Storage Write API into a staging table, then one query job per write; Parquet in GCS, on a table that excludes `_sqlload_source_row`, queried by BigQuery where it lies. See [BigQuery](/docs/targets/bigquery/). |
 | Databricks (beta) | `append`, `merge`, `delete_insert`, `replace` | Parquet in S3 copied where it lies through a Unity Catalog external location; everything else staged as Parquet in a volume, then `COPY INTO`; one statement per write. |
 | ClickHouse, one self-hosted server | `append`, `merge`, `delete_insert`, `replace` | Arrow over HTTP into a staging table, published by swapping in a whole table or whole partitions; Parquet in S3 read by ClickHouse with `s3()`. No clusters, replicated or distributed tables; ClickHouse Cloud is untested. See [ClickHouse](/docs/targets/clickhouse/). |
 | Postgres | `append`, `merge`, `delete_insert`, `replace` | Binary `COPY` into a temporary staging table, then one transaction per commit. See [Postgres](/docs/targets/postgres/). |
diff --git a/website/src/content/docs/docs/targets/bigquery.mdx b/website/src/content/docs/docs/targets/bigquery.mdx
index 792700d77..ceafca25f 100644
--- a/website/src/content/docs/docs/targets/bigquery.mdx
+++ b/website/src/content/docs/docs/targets/bigquery.mdx
@@ -121,6 +121,55 @@ loaded  raw.app__orders     1,000 rows 0.1s
 - **Then one query job.** A query job moves the rows from the staging table into the table. Each
   job carries a request ID, so a retried request never runs it twice.
 
+## Parquet in Google Cloud Storage
+
+Parquet files in a [`gcs`](/docs/configuration/#gcs) bucket can load without passing through
+SQLLoad: BigQuery queries them where they lie, as temporary external tables, into the staging
+table, in one query job per batch (several for a batch of over 1,000 files or a very long query).
+SQLLoad still lists the files, reads their footers and picks the batch; it checks the jobs' row
+count against the footers, and a batch that differs fails.
+
+BigQuery doesn't report a row's position in its file, so it can't fill `_sqlload_source_row`, the
+row's number in its file. A table that keeps that column, as every files table does by default,
+has SQLLoad read the files instead, and `sqlload plan -v 
` says why: + +```console +├── route read parquet into Arrow -> BigQuery takes Arrow +├── why SQLLoad reads the files because BigQuery doesn't report a row's position in its file, so it can't fill _sqlload_source_row; to have BigQuery load the files itself, add _sqlload_source_row to exclude_columns; the destination takes Arrow directly +``` + +To have BigQuery load them, drop the column: + +```toml +[sources.orders] +kind = "files" +connection = "lake" +path = "orders/" +format = "parquet" +exclude_columns = ["_sqlload_source_row"] +``` + +```console +└── route BigQuery reads directly from gs://example-bucket/orders/ (nothing through the loader) +``` + +Without the row number, a file's rows have no order. That matters only to a `merge`, which +doesn't load files into BigQuery yet. + +- **BigQuery reads with its own identity.** The query job reads the bucket as the BigQuery + connection's service account, not with the `gcs` connection's key. Give that service account + `roles/storage.objectViewer` on the bucket. `plan` checks it with a dry run over the first file + to load, which bills nothing, and fails naming the fix when BigQuery can't read it. +- **Every value loads as SQLLoad would load it, or SQLLoad reads the files.** SQLLoad reads them + instead when the table masks columns, when the `gcs` connection sets a custom `endpoint`, when + a column is a UUID, a list, JSON, a [native type](/docs/concepts/types/), or `decimal(20,0)` + (the type of an unsigned 64-bit integer, which BigQuery reads as a signed one), or when a + column's name is one BigQuery would change: anything other than letters, digits and + underscores starting with a letter or underscore, a name starting with a prefix BigQuery + reserves such as `_FILE_`, or two names differing only in case. +- **Files with different columns load together.** BigQuery takes one file's columns for a whole + external table, so each set of files with the same columns gets its own, in the same job. + ## Write strategies Each statement is its own query job and commits on its own. The run record commits after it. @@ -184,6 +233,8 @@ as a UTC instant without its offset. - Records load as `JSON`, not as typed `STRUCT` columns. Columns pinned as [`array(T)`](/docs/concepts/types/#arrays) load as `ARRAY`, except `array(timestamp_tz)`. - `copy_to_internal_stage` is for Snowflake and is refused here. +- BigQuery loads Parquet in GCS itself only on a table that excludes `_sqlload_source_row`; see + [Parquet in Google Cloud Storage](#parquet-in-google-cloud-storage). - A [Postgres CDC](/docs/sources/postgres-cdc/) source can't load into BigQuery; `plan` refuses it: @@ -211,6 +262,11 @@ A live round ran against a real BigQuery project, in one dataset dropped afterwa `contract generate`. - **A failed load:** a Python table that raised after two batches committed no rows and left no staging table. +- **Parquet in GCS:** files in two partition folders, with different columns, loaded both by + BigQuery itself and by SQLLoad's reader, and the two tables matched value for value: decimals + up to 38 digits, timestamps with and without a time zone, dates from year 1, times, floats and + 16-bit integers. An unsigned 64-bit value above BigQuery's `INT64` range loaded through + SQLLoad's reader. - **Deletes:** the key scan and tombstone rounds of `deletes` and `deletes_from`, soft and hard, matched on every step, with a key that can't be NULL and with a text key holding a NULL row. - **Rebuilds:** the statement a type change that rewrites a table renders, such as `int` to diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx index b28b9fd0a..9283798f0 100644 --- a/website/src/content/docs/docs/targets/index.mdx +++ b/website/src/content/docs/docs/targets/index.mdx @@ -10,13 +10,13 @@ names the route for every table. ## How data gets in -| Into | Parquet files in S3 or a folder | CSV, JSON, NDJSON files | Postgres or Python rows | +| Into | Parquet files in S3, GCS or a folder | CSV, JSON, NDJSON files | Postgres or Python rows | |------|---------------------------------|-------------------------|-------------------------| -| DuckDB | DuckDB reads them with `read_parquet` | Read by SQLLoad, appended as Arrow | Appended as Arrow | -| MotherDuck (beta) | In S3: MotherDuck's servers read them with `read_parquet`. In a folder: read by SQLLoad, appended as Arrow | Read by SQLLoad, appended as Arrow | Appended as Arrow | -| Snowflake | `COPY` from an external stage over the bucket | `COPY` from an external stage over the bucket | Written as Parquet to the `load_method`'s stage, then `COPY` (or streamed or inserted, if the `load_method` says so) | +| DuckDB | DuckDB reads them with `read_parquet` (in GCS, with an HMAC key) | Read by SQLLoad, appended as Arrow | Appended as Arrow | +| MotherDuck (beta) | In S3, or in GCS with an HMAC key: MotherDuck's servers read them with `read_parquet`. Otherwise read by SQLLoad, appended as Arrow | Read by SQLLoad, appended as Arrow | Appended as Arrow | +| Snowflake | In S3: `COPY` from an external stage over the bucket. Otherwise read by SQLLoad, then loaded as rows are | In S3: `COPY` from an external stage over the bucket. Otherwise as for Parquet files | Written as Parquet to the `load_method`'s stage, then `COPY` (or streamed or inserted, if the `load_method` says so) | | Databricks (beta) | In S3: `COPY INTO` where they lie, through an external location. In a folder: staged by SQLLoad in the volume, then `COPY INTO` | Read by SQLLoad, staged as Parquet in the volume, then `COPY INTO` | Written as Parquet to the volume, then `COPY INTO` | -| BigQuery (beta) | Read by SQLLoad, streamed as Arrow into a staging table, then one query job | As for Parquet files | Streamed as Arrow into a staging table, then one query job | +| BigQuery (beta) | In GCS, on a table that excludes `_sqlload_source_row`: BigQuery queries them where they lie into a staging table. Otherwise read by SQLLoad, streamed as Arrow into a staging table, then one query job | As for Parquet files | Streamed as Arrow into a staging table, then one query job | | ClickHouse | In S3: ClickHouse reads them with `s3()` into a staging table. In a folder: read by SQLLoad, sent as Arrow | Read by SQLLoad, sent as Arrow into a staging table | Sent as Arrow into a staging table, then swapped in by table or partition | | Postgres | Read by SQLLoad, then binary `COPY` into a temporary staging table | As for Parquet files | Binary `COPY` into a temporary staging table, then one transaction per commit |