diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs index 1fe987921..ae40af621 100644 --- a/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs +++ b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod pull; +pub(crate) mod selected; pub(crate) mod sql_adapter; pub(crate) mod type_names; pub(crate) mod write; diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/dialect/selected.rs b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/selected.rs new file mode 100644 index 000000000..a566872c4 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/_helpers/dialect/selected.rs @@ -0,0 +1,114 @@ +//! The statement that has BigQuery select a BigQuery source's rows itself. + +use crate::adapters::bigquery::models::BigQuerySql; +use crate::adapters::contract::constants::UNLOADED_BATCH_ROWS; +use crate::adapters::contract::models::TableName; +use crate::adapters::contract::types::SqlAdapter; +use crate::column_types::models::ColumnType; +use crate::load_naming::main::metadata_role::metadata_role; +use crate::load_naming::models::{Compat, MetadataRole}; +use crate::postgres_source::models::ReadColumn; +use crate::schema_plan::constants::MERGE_ROW_COLUMN; +use crate::schema_plan::models::DesiredColumn; + +/// One batch of a BigQuery source's rows BigQuery selects itself, into the staging table or, when it `builds`, a build table in the final types. +#[derive(Debug, Clone, Copy)] +pub(crate) struct SelectedLoad<'a> { + pub into: &'a TableName, + pub columns: &'a [DesiredColumn], + /// The source's read columns, each with its quoted name in `query`'s result. + pub read: &'a [ReadColumn], + /// The table's `constant_columns`: name, type and value as text. + pub constants: &'a [(String, ColumnType, String)], + pub compat: Compat, + /// The batch, from 1: a merge's row number counts after earlier batches'. + pub batch: u64, + /// A merge's `unique_key`: its row number counts among the rows sharing a key, which are all its dedupe compares, so BigQuery numbers them in parallel rather than on one worker. + pub merge_key: &'a [String], + pub builds: bool, + pub load_id: &'a str, + pub loaded_at: &'a str, + /// The `SELECT` of the source's rows. + pub query: &'a str, +} + +/// `INSERT INTO into` every one of `columns` from the rows of `query`: a read column by its name in the query's result, a constant from its value, a merge's row number after the earlier batches', among the rows sharing its key, grouped as the merge groups them, in the order the query gives them (a key column the query doesn't read is a constant, the same for every row), the load id and load time the batch's; any other column NULL. Each is cast to its staging or build type without parameters, which a cast can't name and the column then checks; JSON goes into a staging table as its text, as SQLLoad's reader stages it. +pub(crate) fn selected_insert(load: &SelectedLoad<'_>) -> String { + let literal = |text: &str| BigQuerySql.render_loader_value_literal(text); + let mut values: Vec = Vec::with_capacity(load.columns.len()); + for column in load.columns { + let constant = load + .constants + .iter() + .find(|(name, _, _)| *name == column.name) + .map(|(_, _, value)| (literal(value), true)); + let read = load + .read + .iter() + .find(|read| read.name == column.name) + .map(|read| (format!("u.{}", read.expression), false)); + let value = + match ( + metadata_role(load.compat, &column.name), + column.name.as_str(), + ) { + (Some(MetadataRole::LoadId), _) => (literal(load.load_id), true), + (Some(MetadataRole::LoadedAt), _) => (literal(load.loaded_at), true), + (None, MERGE_ROW_COLUMN) => { + let mut keys: Vec = Vec::new(); + for read in load.read { + if load.merge_key.contains(&read.name) { + keys.push(BigQuerySql.render_partition_key( + &format!("u.{}", read.expression), + &read.output, + )); + } + } + let over = match keys.is_empty() { + true => String::new(), + false => format!("PARTITION BY {}", keys.join(", ")), + }; + values.push(format!( + "{} * {UNLOADED_BATCH_ROWS} + ROW_NUMBER() OVER ({over}) - 1", + load.batch + )); + continue; + } + _ => constant + .or(read) + .unwrap_or_else(|| ("NULL".to_owned(), true)), + }; + values.push(typed(value, &column.column_type, load.builds)); + } + let targets: Vec = load + .columns + .iter() + .map(|column| BigQuerySql.render_identifier(&column.name)) + .collect(); + format!( + "INSERT INTO {} ({}) SELECT {} FROM ({}) AS u", + BigQuerySql.render_qualified_name(load.into), + targets.join(", "), + values.join(", "), + load.query + ) +} + +/// `value` as a column of `column_type` holds it in the staging table, or in a build table (`builds`): JSON as its text in staging and as JSON in a build table (parsed when the value is text, `from_text`: a literal or NULL), anything else cast to its type without parameters. +fn typed((value, from_text): (String, bool), column_type: &ColumnType, builds: bool) -> String { + match (column_type, builds, from_text) { + (ColumnType::Json, false, false) => format!("TO_JSON_STRING({value})"), + (ColumnType::Json, true, false) => value, + (ColumnType::Json, true, true) => format!("PARSE_JSON({value})"), + _ => { + let sql_type = match builds { + true => BigQuerySql.render_loader_logical_type(column_type).sql, + false => BigQuerySql.render_staging_type(column_type), + }; + let base = sql_type + .split_once('(') + .map_or(sql_type.as_str(), |(base, _)| base); + format!("CAST({value} AS {base})") + } + } +} diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs b/crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs index a3a8db791..5bbb6c05c 100644 --- a/crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs +++ b/crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs @@ -1,18 +1,15 @@ //! A files batch loaded by BigQuery querying the source's own Parquet files in GCS. -use serde_json::{Value, json}; - use crate::adapters::bigquery::_helpers::dialect::pull::{ PulledFile, PulledGroup, pull_queries, unread_name, }; +use crate::adapters::bigquery::_helpers::write::insert_job::insert_job; use crate::adapters::bigquery::main::plan::_direct_problem::direct_problem; use crate::adapters::bigquery::models::BigQueryWriter; use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem; use crate::adapters::contract::main::_footer_rows::footer_rows; use crate::adapters::contract::models::{DirectBatch, DirectLoaded, TableName}; use crate::adapters::contract::types::TableWriter; -use crate::bigquery_api::main::load_job::load_job; -use crate::bigquery_api::models::BigQueryClient; use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN}; use crate::files_source::main::normal_value::normal_value; use crate::files_source::models::{BatchFooters, DataFile}; @@ -66,7 +63,13 @@ impl BigQueryWriter { ); let mut rows = 0; for (query, definitions) in queries { - rows += query_files(&self.destination.api, query, definitions).await?; + rows += insert_job( + &self.destination.api, + &query, + Some(definitions), + "this batch's files from GCS", + ) + .await?; } let expected = footer_rows(batch.footers); if rows != expected { @@ -163,40 +166,3 @@ fn footer_fields(footers: &BatchFooters, data: &DataFile) -> Vec { .map(|field| field.name().clone()) .collect() } - -/// Run one query job inserting files in GCS, its temporary external tables `definitions`: the rows it inserted. A job BigQuery fails, or whose count is not a number, fails the batch. -async fn query_files( - api: &BigQueryClient, - query: String, - definitions: Value, -) -> Result { - let job = json!({ - "jobReference": { - "projectId": api.project, - "jobId": format!("sqlload_files_{}", uuid::Uuid::new_v4().simple()), - "location": api.location, - }, - "configuration": {"query": { - "query": query, - "useLegacySql": false, - "tableDefinitions": definitions, - }}, - }); - let resource = load_job(api, &job, None).await?; - let error = &resource["status"]["errorResult"]; - if !error.is_null() { - return Err(LoaderError::Invalid(format!( - "BigQuery could not load this batch's files from GCS: {}: {}", - error["reason"].as_str().unwrap_or_default(), - error["message"].as_str().unwrap_or_default() - ))); - } - let affected = resource["statistics"]["query"]["numDmlAffectedRows"] - .as_str() - .unwrap_or_default(); - affected.parse().map_err(|_| { - LoaderError::Invalid(format!( - "BigQuery's query job reported {affected:?} inserted rows, not a count; the batch is not committed" - )) - }) -} diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/write/insert_job.rs b/crates/sqlload/src/adapters/bigquery/_helpers/write/insert_job.rs new file mode 100644 index 000000000..e6ebc3048 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/_helpers/write/insert_job.rs @@ -0,0 +1,45 @@ +//! One `INSERT ... SELECT` BigQuery runs as a query job, counting the rows it inserted. + +use serde_json::{Value, json}; + +use crate::bigquery_api::main::load_job::load_job; +use crate::bigquery_api::models::BigQueryClient; +use crate::loader::errors::LoaderError; + +/// Run `query` as one query job, with its temporary external tables `definitions` when it reads files: the rows it inserted. A job BigQuery fails, or whose count is not a number, fails the batch; `what` names what the job loads in the error. +pub(in crate::adapters::bigquery) async fn insert_job( + api: &BigQueryClient, + query: &str, + definitions: Option, + what: &str, +) -> Result { + let mut configuration = json!({"query": query, "useLegacySql": false}); + if let Some(definitions) = definitions { + configuration["tableDefinitions"] = definitions; + } + let job = json!({ + "jobReference": { + "projectId": api.project, + "jobId": format!("sqlload_insert_{}", uuid::Uuid::new_v4().simple()), + "location": api.location, + }, + "configuration": {"query": configuration}, + }); + let resource = load_job(api, &job, None).await?; + let error = &resource["status"]["errorResult"]; + if !error.is_null() { + return Err(LoaderError::Invalid(format!( + "BigQuery could not load {what}: {}: {}", + error["reason"].as_str().unwrap_or_default(), + error["message"].as_str().unwrap_or_default() + ))); + } + let affected = resource["statistics"]["query"]["numDmlAffectedRows"] + .as_str() + .unwrap_or_default(); + affected.parse().map_err(|_| { + LoaderError::Invalid(format!( + "BigQuery's query job reported {affected:?} inserted rows, not a count; the batch is not committed" + )) + }) +} diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs b/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs index 4254929b4..a5153a441 100644 --- a/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs +++ b/crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod arrow_rows; pub(crate) mod direct_load; +pub(crate) mod insert_job; +pub(crate) mod selected_load; pub(crate) mod table_writer; diff --git a/crates/sqlload/src/adapters/bigquery/_helpers/write/selected_load.rs b/crates/sqlload/src/adapters/bigquery/_helpers/write/selected_load.rs new file mode 100644 index 000000000..b8f92b185 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/_helpers/write/selected_load.rs @@ -0,0 +1,52 @@ +//! A BigQuery source's rows selected by BigQuery itself, on the connection the target loads through. + +use crate::adapters::bigquery::_helpers::dialect::selected::{SelectedLoad, selected_insert}; +use crate::adapters::bigquery::_helpers::write::insert_job::insert_job; +use crate::adapters::bigquery::models::BigQueryWriter; +use crate::adapters::contract::models::{DirectLoaded, TableName, UnloadedBatch, UnloadedSource}; +use crate::adapters::contract::types::TableWriter; +use crate::loader::errors::LoaderError; + +impl BigQueryWriter { + /// Load `batch` by having BigQuery select the source's rows into the staging or build table the load publishes from, in one query job. Only a query, which plan gives a BigQuery source on the target's own connection, is loaded this way: BigQuery never reads unloaded files. + pub(in crate::adapters) async fn pull_unloaded( + &mut self, + batch: &UnloadedBatch<'_>, + ) -> Result { + let UnloadedSource::Query(query) = batch.source else { + return Err(LoaderError::Invalid( + "BigQuery does not load unloaded files; the plan should have had SQLLoad read them" + .to_owned(), + )); + }; + self.settle_rows().await?; + let target = &batch.target; + let (into, columns) = match (target.staging, target.builds) { + (Some(staging), false) => (staging, target.columns), + (staging, _) => (staging.unwrap_or(target.table), batch.write_columns), + }; + let into = TableName { + database: Some( + into.database + .clone() + .unwrap_or_else(|| self.destination.project.clone()), + ), + ..into.clone() + }; + let insert = selected_insert(&SelectedLoad { + into: &into, + columns, + read: batch.read, + constants: target.constants, + compat: target.compat, + batch: target.batch, + merge_key: batch.merge_key, + builds: target.builds, + load_id: target.load_id, + loaded_at: batch.loaded_at, + query, + }); + let rows = insert_job(&self.destination.api, &insert, None, "the source's rows").await?; + Ok(DirectLoaded::Staged(rows)) + } +} diff --git a/crates/sqlload/src/adapters/bigquery/tests.rs b/crates/sqlload/src/adapters/bigquery/tests.rs index 4852a7487..2b7f0e655 100644 --- a/crates/sqlload/src/adapters/bigquery/tests.rs +++ b/crates/sqlload/src/adapters/bigquery/tests.rs @@ -6,5 +6,7 @@ mod dialect; mod helpers; #[path = "tests/test_pull_query.rs"] mod pull_query; +#[path = "tests/test_selected_insert.rs"] +mod selected_insert; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlload/src/adapters/bigquery/tests/test_selected_insert.rs b/crates/sqlload/src/adapters/bigquery/tests/test_selected_insert.rs new file mode 100644 index 000000000..f663539a3 --- /dev/null +++ b/crates/sqlload/src/adapters/bigquery/tests/test_selected_insert.rs @@ -0,0 +1,105 @@ +use crate::adapters::bigquery::_helpers::dialect::selected::{SelectedLoad, selected_insert}; +use crate::adapters::bigquery::tests::helpers::{desired, orders}; +use crate::adapters::bigquery::tests::test_types::SelectedInsertTestCase; +use crate::column_types::models::ColumnType; +use crate::load_naming::models::Compat; +use crate::postgres_source::models::{ReadColumn, WireType}; + +#[test] +fn given_a_bigquery_source_on_the_targets_connection_when_bigquery_selects_its_rows_itself_then_each_column_comes_from_the_right_place() + { + let read = [ + ReadColumn { + name: "id".to_owned(), + expression: "`Order ID`".to_owned(), + wire: WireType::Text, + output: ColumnType::Int, + }, + ReadColumn { + name: "amount".to_owned(), + expression: "`amount`".to_owned(), + wire: WireType::Text, + output: ColumnType::Decimal { + precision: 12, + scale: 2, + }, + }, + ReadColumn { + name: "score".to_owned(), + expression: "`score`".to_owned(), + wire: WireType::Text, + output: ColumnType::Float, + }, + ReadColumn { + name: "doc".to_owned(), + expression: "`doc`".to_owned(), + wire: WireType::Text, + output: ColumnType::Json, + }, + ]; + let constants = [("region".to_owned(), ColumnType::Varchar(4), "eu".to_owned())]; + let columns = vec![ + desired("id", ColumnType::Int), + desired( + "amount", + ColumnType::Decimal { + precision: 12, + scale: 2, + }, + ), + desired("doc", ColumnType::Json), + desired("region", ColumnType::Varchar(4)), + desired("gone", ColumnType::Text), + desired("_sqlload_row", ColumnType::Int), + desired("_sqlload_load_id", ColumnType::Text), + desired("_sqlload_loaded_at", ColumnType::TimestampTz), + ]; + let test_cases = [ + SelectedInsertTestCase { + description: "into the staging table: each read column by its name in the source's query, cast without parameters, JSON as its text, a constant from its value, a column the source lacks NULL, a merge's row number after the earlier batches' among the rows sharing its key (a constant key column groups every row alike), and the batch's load id and time", + columns: columns.clone(), + builds: false, + merge_key: vec!["id", "region"], + expected_insert: "INSERT INTO `p`.`raw`.`orders` (`id`, `amount`, `doc`, `region`, `gone`, `_sqlload_row`, `_sqlload_load_id`, `_sqlload_loaded_at`) SELECT CAST(u.`Order ID` AS INT64), CAST(u.`amount` AS NUMERIC), TO_JSON_STRING(u.`doc`), CAST('eu' AS STRING), CAST(NULL AS STRING), 3 * 281474976710656 + ROW_NUMBER() OVER (PARTITION BY u.`Order ID`) - 1, CAST('L1' AS STRING), CAST('2026-11-06T00:00:00Z' AS TIMESTAMP) FROM (SELECT `Order ID`, `amount`, `doc` FROM `p.marts.orders`) AS u", + }, + SelectedInsertTestCase { + description: "into a build table JSON stays JSON", + columns: vec![desired("doc", ColumnType::Json)], + builds: true, + merge_key: Vec::new(), + expected_insert: "INSERT INTO `p`.`raw`.`orders` (`doc`) SELECT u.`doc` FROM (SELECT `Order ID`, `amount`, `doc` FROM `p.marts.orders`) AS u", + }, + SelectedInsertTestCase { + description: "a FLOAT64 key, which BigQuery refuses in PARTITION BY, numbers the rows by its text, as the merge itself groups them", + columns: vec![desired("_sqlload_row", ColumnType::Int)], + builds: false, + merge_key: vec!["score"], + expected_insert: "INSERT INTO `p`.`raw`.`orders` (`_sqlload_row`) SELECT 3 * 281474976710656 + ROW_NUMBER() OVER (PARTITION BY IF(IS_NAN(u.`score`), GENERATE_UUID(), FORMAT('%.17g', u.`score` + 0.0))) - 1 FROM (SELECT `Order ID`, `amount`, `doc` FROM `p.marts.orders`) AS u", + }, + ]; + for test_case in test_cases { + let merge_key: Vec = test_case + .merge_key + .iter() + .map(|key| (*key).to_owned()) + .collect(); + let insert = selected_insert(&SelectedLoad { + into: &orders(), + columns: &test_case.columns, + read: &read, + constants: &constants, + compat: Compat::Sqlload, + batch: 3, + merge_key: &merge_key, + builds: test_case.builds, + load_id: "L1", + loaded_at: "2026-11-06T00:00:00Z", + query: "SELECT `Order ID`, `amount`, `doc` FROM `p.marts.orders`", + }); + assert_eq!( + insert, test_case.expected_insert, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/bigquery/tests/test_types.rs b/crates/sqlload/src/adapters/bigquery/tests/test_types.rs index 5668ba7bb..0a3e3e97a 100644 --- a/crates/sqlload/src/adapters/bigquery/tests/test_types.rs +++ b/crates/sqlload/src/adapters/bigquery/tests/test_types.rs @@ -43,3 +43,13 @@ pub(crate) struct PullQueriesTestCase { pub(crate) groups: Vec, pub(crate) expected_queries: usize, } + +pub(crate) struct SelectedInsertTestCase { + pub(crate) description: &'static str, + pub(crate) columns: Vec, + /// The rows go to a build table in their final types, rather than the staging table. + pub(crate) builds: bool, + /// A merge's `unique_key`. + pub(crate) merge_key: Vec<&'static str>, + pub(crate) expected_insert: &'static str, +} diff --git a/crates/sqlload/src/adapters/contract/models.rs b/crates/sqlload/src/adapters/contract/models.rs index 0ac0aa204..c8e542147 100644 --- a/crates/sqlload/src/adapters/contract/models.rs +++ b/crates/sqlload/src/adapters/contract/models.rs @@ -287,6 +287,8 @@ pub struct DirectBatch<'a> { #[derive(Debug, Clone, Copy)] pub struct UnloadedBatch<'a> { pub source: UnloadedSource<'a>, + /// A merge's `unique_key`; empty for any other write mode. + pub merge_key: &'a [String], /// The columns the rows hold, as they are read. pub read: &'a [ReadColumn], /// The table's columns as the load writes them. diff --git a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs index 10d762c0e..76b61e470 100644 --- a/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs +++ b/crates/sqlload/src/adapters/dispatch/main/write/load_unloaded.rs @@ -15,6 +15,7 @@ pub async fn load_unloaded( Writer::Snowflake(snowflake) => snowflake.pull_unloaded(batch).await.map(Some), Writer::ClickHouse(clickhouse) => clickhouse.pull_unloaded(batch).await.map(Some), Writer::Databricks(databricks) => databricks.pull_unloaded(batch).await.map(Some), + Writer::BigQuery(bigquery) => bigquery.pull_unloaded(batch).await.map(Some), _ => Ok(None), } } diff --git a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs index 844426e67..20b4f5fb8 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/helpers.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/helpers.rs @@ -157,6 +157,7 @@ pub(crate) fn pulled_unloaded(reported: u64, with_files: bool) -> Result<(String }; let batch = UnloadedBatch { source: crate::adapters::contract::models::UnloadedSource::Files(&files), + merge_key: &[], read: &read, write_columns: &columns, target: WriteTarget { diff --git a/crates/sqlload/src/bigquery_source/_helpers/catalog.rs b/crates/sqlload/src/bigquery_source/_helpers/catalog.rs index 716a983b9..c37bbc57e 100644 --- a/crates/sqlload/src/bigquery_source/_helpers/catalog.rs +++ b/crates/sqlload/src/bigquery_source/_helpers/catalog.rs @@ -58,7 +58,7 @@ async fn described( (dataset, table): (String, String), ) -> Result, BigQuerySourceError> { let found = table_metadata(&reader.client, (&reader.project, &dataset, &table)).await?; - Ok(found.map(|found| listed_table((dataset, table), &found))) + Ok(found.map(|found| listed_table((dataset, table), &found, &reader.client.location))) } /// One glob set for `dataset.table` patterns, matched exactly. @@ -75,8 +75,12 @@ fn matcher(patterns: &[String]) -> Result { .map_err(|error| BigQuerySourceError::Invalid(error.to_string())) } -/// `dataset.table` from `found`: its columns, declared primary key, row and byte counts, and description. Anything the Storage Read API cannot read directly (a view) has `estimated_bytes` -1, so it is read through a query. -pub(crate) fn listed_table((dataset, table): (String, String), found: &Value) -> SourceTable { +/// `dataset.table` from `found`: its columns, declared primary key, row and byte counts, description, and its location when query jobs run in another (`job_location`). Anything the Storage Read API cannot read directly (a view) has `estimated_bytes` -1, so it is read through a query. +pub(crate) fn listed_table( + (dataset, table): (String, String), + found: &Value, + job_location: &str, +) -> SourceTable { let mut columns: Vec = Vec::new(); for field in found["schema"]["fields"].as_array().into_iter().flatten() { columns.push(source_column(field)); @@ -104,6 +108,10 @@ pub(crate) fn listed_table((dataset, table): (String, String), found: &Value) -> }, comment: found["description"].as_str().map(str::to_owned), listed_keys: Vec::new(), + other_location: found["location"] + .as_str() + .filter(|location| !location.eq_ignore_ascii_case(job_location)) + .map(str::to_owned), } } diff --git a/crates/sqlload/src/bigquery_source/_helpers/query.rs b/crates/sqlload/src/bigquery_source/_helpers/query.rs index 820b9285e..40601a6aa 100644 --- a/crates/sqlload/src/bigquery_source/_helpers/query.rs +++ b/crates/sqlload/src/bigquery_source/_helpers/query.rs @@ -8,8 +8,12 @@ pub(crate) fn quote_identifier(name: &str) -> String { format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")) } -/// `SELECT FROM [WHERE ]`; the table is `project.dataset.table` in one pair of backticks, and the query a `subquery`. -pub(crate) fn select_query(project: &str, request: &BigQueryReadRequest<'_>) -> String { +/// `SELECT FROM [FOR SYSTEM_TIME AS OF ...] [WHERE ]`; the table is `project.dataset.table` in one pair of backticks, read as of `as_of` (milliseconds) when given, and the query a `subquery`. +pub(crate) fn select_query( + project: &str, + request: &BigQueryReadRequest<'_>, + as_of: Option, +) -> String { let columns: Vec<&str> = request .columns .iter() @@ -22,11 +26,14 @@ pub(crate) fn select_query(project: &str, request: &BigQueryReadRequest<'_>) -> request.table.schema, request.table.name )), }; + let instant = as_of + .map(|millis| format!(" FOR SYSTEM_TIME AS OF TIMESTAMP_MILLIS({millis})")) + .unwrap_or_default(); let filter = request .filter .map(|filter| format!(" WHERE {filter}")) .unwrap_or_default(); - format!("SELECT {} FROM {from}{filter}", columns.join(", ")) + format!("SELECT {} FROM {from}{instant}{filter}", columns.join(", ")) } /// `SELECT COUNT(*) FROM WHERE `, the table and query written as `select_query` writes them: how many rows `filter` selects. diff --git a/crates/sqlload/src/bigquery_source/_helpers/session.rs b/crates/sqlload/src/bigquery_source/_helpers/session.rs index 52ba8222d..012c858db 100644 --- a/crates/sqlload/src/bigquery_source/_helpers/session.rs +++ b/crates/sqlload/src/bigquery_source/_helpers/session.rs @@ -34,7 +34,11 @@ pub(crate) async fn read_session( Some(reader.snapshot_ms), ), false => ( - query_destination(&reader.client, &select_query(&reader.project, request)).await?, + query_destination( + &reader.client, + &select_query(&reader.project, request, None), + ) + .await?, None, None, ), diff --git a/crates/sqlload/src/bigquery_source/constants.rs b/crates/sqlload/src/bigquery_source/constants.rs index d0e49a88c..58020aafb 100644 --- a/crates/sqlload/src/bigquery_source/constants.rs +++ b/crates/sqlload/src/bigquery_source/constants.rs @@ -15,3 +15,6 @@ pub const READ_API_TABLES: &[&str] = &["TABLE"]; /// The precision and scale of a `NUMERIC` column declared without them. pub const NUMERIC_DEFAULT: (u8, u8) = (38, 9); + +/// BigQuery types SQLLoad's reader converts: records and ranges to JSON, geographies to their WKT text. +pub const CONVERTED_TYPES: [&str; 3] = ["STRUCT", "RANGE", "GEOGRAPHY"]; diff --git a/crates/sqlload/src/bigquery_source/main/mod.rs b/crates/sqlload/src/bigquery_source/main/mod.rs index 7817a5f2c..d50a9b8d5 100644 --- a/crates/sqlload/src/bigquery_source/main/mod.rs +++ b/crates/sqlload/src/bigquery_source/main/mod.rs @@ -4,3 +4,5 @@ pub mod dry_count; pub mod read_catalog; pub mod read_columns; pub mod read_rows; +pub mod select_problem; +pub mod selected_rows_query; diff --git a/crates/sqlload/src/bigquery_source/main/read_catalog.rs b/crates/sqlload/src/bigquery_source/main/read_catalog.rs index 89fe3a091..088bf03f3 100644 --- a/crates/sqlload/src/bigquery_source/main/read_catalog.rs +++ b/crates/sqlload/src/bigquery_source/main/read_catalog.rs @@ -37,6 +37,7 @@ pub async fn read_catalog( estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, }); } Ok(tables) diff --git a/crates/sqlload/src/bigquery_source/main/select_problem.rs b/crates/sqlload/src/bigquery_source/main/select_problem.rs new file mode 100644 index 000000000..760adb197 --- /dev/null +++ b/crates/sqlload/src/bigquery_source/main/select_problem.rs @@ -0,0 +1,43 @@ +//! Whether a BigQuery target can select a BigQuery table's rows itself. + +use std::collections::BTreeMap; + +use crate::bigquery_source::constants::CONVERTED_TYPES; +use crate::column_types::models::ColumnType; +use crate::postgres_source::models::SourceTable; + +/// Why a BigQuery target writing through the source's own connection can't select `table`'s rows itself, exactly as SQLLoad's reader would load them, with the `pinned` types: a table in another location than its connection's query jobs, which only the Storage Read API reaches; a column BigQuery doesn't hold as it loads (a record, a repeated column, a range or a geography, which SQLLoad converts to JSON or text), or a column pinned to another type than its own, which SQLLoad converts; `None` when every column copies as it is. +pub fn select_problem( + table: &SourceTable, + pinned: &BTreeMap, +) -> Option { + if let Some(location) = &table.other_location { + return Some(format!( + "the table is in {location}, not the location query jobs on its connection run in" + )); + } + table.columns.iter().find_map(|column| { + let held = column.postgres_type.as_str(); + let plain = !held.starts_with("ARRAY<") && !CONVERTED_TYPES.contains(&held); + let output = match (pinned.get(&column.name), &column.column_type) { + (Some(pinned), _) => Some(pinned), + (None, Ok(natural)) => Some(natural), + (None, Err(_)) => None, + }; + match (plain, output, &column.column_type) { + (_, None, _) => Some(format!( + "{} is BigQuery {held}, which has no type SQLLoad loads", + column.name + )), + (false, Some(output), _) => Some(format!( + "{} is BigQuery {held}, which SQLLoad converts to {output}", + column.name + )), + (true, Some(output), Ok(natural)) if output == natural => None, + (true, Some(output), _) => Some(format!( + "{} is pinned as {output}, which SQLLoad converts from BigQuery {held}", + column.name + )), + } + }) +} diff --git a/crates/sqlload/src/bigquery_source/main/selected_rows_query.rs b/crates/sqlload/src/bigquery_source/main/selected_rows_query.rs new file mode 100644 index 000000000..bc8510afe --- /dev/null +++ b/crates/sqlload/src/bigquery_source/main/selected_rows_query.rs @@ -0,0 +1,14 @@ +//! The query a destination writing through a BigQuery source's own connection runs to select its rows itself. + +use crate::bigquery_source::_helpers::query::select_query; +use crate::bigquery_source::models::{BigQueryReadRequest, BigQueryReader}; + +/// The `SELECT` of `request`'s read columns that the Storage Read API would read: a table read directly is read as of the reader's instant (`FOR SYSTEM_TIME AS OF`), so every part of one load sees the same rows; a view, a query table or a table with a `where` is queried as SQLLoad's own read queries it. +pub fn selected_rows_query(reader: &BigQueryReader, request: &BigQueryReadRequest<'_>) -> String { + let direct = request.sql.is_none() && request.table.estimated_bytes >= 0 && !request.filtered; + select_query( + &reader.project, + request, + direct.then_some(reader.snapshot_ms), + ) +} diff --git a/crates/sqlload/src/bigquery_source/tests.rs b/crates/sqlload/src/bigquery_source/tests.rs index 5a8881cdd..3cbdd5f9b 100644 --- a/crates/sqlload/src/bigquery_source/tests.rs +++ b/crates/sqlload/src/bigquery_source/tests.rs @@ -6,6 +6,8 @@ mod count_query; mod helpers; #[path = "tests/test_read_columns.rs"] mod read_columns; +#[path = "tests/test_select_problem.rs"] +mod select_problem; #[path = "tests/test_select_query.rs"] mod select_query; #[path = "tests/test_types.rs"] diff --git a/crates/sqlload/src/bigquery_source/tests/helpers.rs b/crates/sqlload/src/bigquery_source/tests/helpers.rs index 877745c66..125b63c38 100644 --- a/crates/sqlload/src/bigquery_source/tests/helpers.rs +++ b/crates/sqlload/src/bigquery_source/tests/helpers.rs @@ -60,5 +60,6 @@ pub(crate) fn orders() -> SourceTable { split_pages: None, comment: None, listed_keys: Vec::new(), + other_location: None, } } diff --git a/crates/sqlload/src/bigquery_source/tests/test_select_problem.rs b/crates/sqlload/src/bigquery_source/tests/test_select_problem.rs new file mode 100644 index 000000000..ad8aca187 --- /dev/null +++ b/crates/sqlload/src/bigquery_source/tests/test_select_problem.rs @@ -0,0 +1,105 @@ +use serde_json::json; + +use crate::bigquery_source::_helpers::catalog::listed_table; +use crate::bigquery_source::main::select_problem::select_problem; +use crate::bigquery_source::tests::test_types::SelectProblemTestCase; +use crate::column_types::models::ColumnType; + +#[test] +fn given_a_bigquery_table_when_a_bigquery_target_could_select_it_itself_then_only_columns_it_holds_as_they_are_allow_it() + { + let test_cases = [ + SelectProblemTestCase { + description: "scalar columns read as their natural types copy exactly", + fields: vec![ + json!({"name": "id", "type": "INT64", "mode": "REQUIRED"}), + json!({"name": "amount", "type": "NUMERIC", "precision": "12", "scale": "2"}), + json!({"name": "big", "type": "BIGNUMERIC", "precision": "38", "scale": "2"}), + json!({"name": "at", "type": "TIMESTAMP"}), + json!({"name": "local", "type": "DATETIME"}), + json!({"name": "doc", "type": "JSON"}), + json!({"name": "note", "type": "STRING", "maxLength": "10"}), + json!({"name": "raw", "type": "BYTES"}), + ], + pinned: Vec::new(), + location: "EU", + expected_problem: None, + }, + SelectProblemTestCase { + description: "a repeated column, which SQLLoad reads as JSON, keeps SQLLoad's reader", + fields: vec![ + json!({"name": "id", "type": "INT64"}), + json!({"name": "tags", "type": "STRING", "mode": "REPEATED"}), + ], + pinned: Vec::new(), + location: "EU", + expected_problem: Some( + "tags is BigQuery ARRAY, which SQLLoad converts to json", + ), + }, + SelectProblemTestCase { + description: "a record keeps SQLLoad's reader", + fields: vec![json!({"name": "address", "type": "RECORD"})], + pinned: Vec::new(), + location: "EU", + expected_problem: Some("address is BigQuery STRUCT, which SQLLoad converts to json"), + }, + SelectProblemTestCase { + description: "geography, read as its WKT text, keeps SQLLoad's reader", + fields: vec![json!({"name": "place", "type": "GEOGRAPHY"})], + pinned: Vec::new(), + location: "EU", + expected_problem: Some("place is BigQuery GEOGRAPHY, which SQLLoad converts to text"), + }, + SelectProblemTestCase { + description: "a column pinned to another type keeps SQLLoad's reader, which converts it", + fields: vec![json!({"name": "id", "type": "INT64"})], + pinned: vec![("id", ColumnType::Text)], + location: "EU", + expected_problem: Some( + "id is pinned as text, which SQLLoad converts from BigQuery INT64", + ), + }, + SelectProblemTestCase { + description: "a pin to the column's own type changes nothing", + fields: vec![json!({"name": "id", "type": "INT64"})], + pinned: vec![("id", ColumnType::Int)], + location: "EU", + expected_problem: None, + }, + SelectProblemTestCase { + description: "a table in another location than the connection's query jobs keeps SQLLoad's reader, which reads it from anywhere", + fields: vec![json!({"name": "id", "type": "INT64"})], + pinned: Vec::new(), + location: "US", + expected_problem: Some( + "the table is in US, not the location query jobs on its connection run in", + ), + }, + SelectProblemTestCase { + description: "a location written in another case is the same location", + fields: vec![json!({"name": "id", "type": "INT64"})], + pinned: Vec::new(), + location: "eu", + expected_problem: None, + }, + ]; + for test_case in test_cases { + let table = listed_table( + ("shop".to_owned(), "orders".to_owned()), + &json!({"type": "TABLE", "location": test_case.location, "schema": {"fields": test_case.fields}}), + "EU", + ); + let pinned = test_case + .pinned + .iter() + .map(|(name, column_type)| ((*name).to_owned(), column_type.clone())) + .collect(); + assert_eq!( + select_problem(&table, &pinned).as_deref(), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/bigquery_source/tests/test_select_query.rs b/crates/sqlload/src/bigquery_source/tests/test_select_query.rs index 52526281d..aec50c4ec 100644 --- a/crates/sqlload/src/bigquery_source/tests/test_select_query.rs +++ b/crates/sqlload/src/bigquery_source/tests/test_select_query.rs @@ -4,31 +4,45 @@ use crate::bigquery_source::tests::helpers::orders; use crate::bigquery_source::tests::test_types::SelectQueryTestCase; #[test] -fn given_a_query_table_when_wrapping_it_with_a_cursor_filter_then_the_filter_stays_outside_the_query() +fn given_a_table_or_query_when_selecting_it_with_a_cursor_filter_then_the_filter_stays_outside_and_an_instant_follows_the_table() { let table = orders(); let test_cases = [ SelectQueryTestCase { description: "a final semicolon is dropped and a trailing comment ends before the parenthesis", - sql: " SELECT 1 AS id -- latest only\n;\n", + sql: Some(" SELECT 1 AS id -- latest only\n;\n"), + as_of: None, expected_query: "SELECT FROM (SELECT 1 AS id -- latest only\n) WHERE `id` > 3", }, SelectQueryTestCase { description: "a plain query is wrapped as it is", - sql: "SELECT id FROM marts.orders", + sql: Some("SELECT id FROM marts.orders"), + as_of: None, expected_query: "SELECT FROM (SELECT id FROM marts.orders\n) WHERE `id` > 3", }, + SelectQueryTestCase { + description: "a table read as of an instant names it right after the table, before the filter", + sql: None, + as_of: Some(1_763_500_000_000), + expected_query: "SELECT FROM `p.marts.orders` FOR SYSTEM_TIME AS OF TIMESTAMP_MILLIS(1763500000000) WHERE `id` > 3", + }, + SelectQueryTestCase { + description: "a table read as it is now names no instant", + sql: None, + as_of: None, + expected_query: "SELECT FROM `p.marts.orders` WHERE `id` > 3", + }, ]; for test_case in test_cases { let request = BigQueryReadRequest { table: &table, - sql: Some(test_case.sql), + sql: test_case.sql, columns: &[], filter: Some("`id` > 3"), filtered: false, }; assert_eq!( - select_query("p", &request), + select_query("p", &request, test_case.as_of), test_case.expected_query, "{}", test_case.description diff --git a/crates/sqlload/src/bigquery_source/tests/test_types.rs b/crates/sqlload/src/bigquery_source/tests/test_types.rs index 80ed35041..ba86962cc 100644 --- a/crates/sqlload/src/bigquery_source/tests/test_types.rs +++ b/crates/sqlload/src/bigquery_source/tests/test_types.rs @@ -30,8 +30,10 @@ pub(crate) struct ReadColumnsTestCase { pub(crate) struct SelectQueryTestCase { pub(crate) description: &'static str, - /// The query table's `sql`. - pub(crate) sql: &'static str, + /// The query table's `sql`; `None` reads the table itself. + pub(crate) sql: Option<&'static str>, + /// The instant, in milliseconds, a table is read as of. + pub(crate) as_of: Option, pub(crate) expected_query: &'static str, } @@ -42,3 +44,14 @@ pub(crate) struct CountQueryTestCase { pub(crate) filter: &'static str, pub(crate) expected_query: &'static str, } + +pub(crate) struct SelectProblemTestCase { + pub(crate) description: &'static str, + /// The table's columns, as `tables.get` describes them. + pub(crate) fields: Vec, + pub(crate) pinned: Vec<(&'static str, ColumnType)>, + /// The table's location, as `tables.get` gives it; query jobs on its connection run in `EU`. + pub(crate) location: &'static str, + /// Why BigQuery can't select the rows into a BigQuery table itself, if it can't. + pub(crate) expected_problem: Option<&'static str>, +} diff --git a/crates/sqlload/src/contract/tests/helpers.rs b/crates/sqlload/src/contract/tests/helpers.rs index fe139f67b..39aab5d3b 100644 --- a/crates/sqlload/src/contract/tests/helpers.rs +++ b/crates/sqlload/src/contract/tests/helpers.rs @@ -66,6 +66,7 @@ fn orders(_: &crate::project::models::SourceConfig) -> Vec { split_pages: None, comment: None, listed_keys: Vec::new(), + other_location: None, }] } diff --git a/crates/sqlload/src/databricks_source/_helpers/catalog.rs b/crates/sqlload/src/databricks_source/_helpers/catalog.rs index 54067e4b7..9c9d608ba 100644 --- a/crates/sqlload/src/databricks_source/_helpers/catalog.rs +++ b/crates/sqlload/src/databricks_source/_helpers/catalog.rs @@ -55,6 +55,7 @@ pub(crate) async fn catalog_tables( estimated_bytes: -1, comment: text(row, 2).filter(|comment| !comment.is_empty()), listed_keys: Vec::new(), + other_location: None, }, ); } diff --git a/crates/sqlload/src/databricks_source/main/read_catalog.rs b/crates/sqlload/src/databricks_source/main/read_catalog.rs index c421072f6..04f22974e 100644 --- a/crates/sqlload/src/databricks_source/main/read_catalog.rs +++ b/crates/sqlload/src/databricks_source/main/read_catalog.rs @@ -44,6 +44,7 @@ pub async fn read_catalog( estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, }); } Ok(tables) diff --git a/crates/sqlload/src/databricks_source/tests/test_select_query.rs b/crates/sqlload/src/databricks_source/tests/test_select_query.rs index 9564a2e37..81fd8dd64 100644 --- a/crates/sqlload/src/databricks_source/tests/test_select_query.rs +++ b/crates/sqlload/src/databricks_source/tests/test_select_query.rs @@ -22,6 +22,7 @@ fn given_a_table_or_query_when_selecting_it_then_names_are_backquoted_in_the_cat estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, }; let columns = [ReadColumn { name: "id".to_owned(), @@ -100,6 +101,7 @@ fn given_a_string_column_when_reading_it_then_a_non_binary_collation_is_compared estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, }; let columns = read_columns(&table, &BTreeMap::new()).unwrap_or_default(); let expression = columns.first().map(|column| column.expression.as_str()); diff --git a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs index cf6158dc8..c3b75c91b 100644 --- a/crates/sqlload/src/duckdb_source/_helpers/catalog.rs +++ b/crates/sqlload/src/duckdb_source/_helpers/catalog.rs @@ -46,6 +46,7 @@ pub(crate) fn catalog_tables( estimated_bytes: -1, comment: Some(text(row, 2)).filter(|comment| !comment.is_empty()), listed_keys: Vec::new(), + other_location: None, }, ); } diff --git a/crates/sqlload/src/duckdb_source/main/read_catalog.rs b/crates/sqlload/src/duckdb_source/main/read_catalog.rs index 7b87ab0f8..870011f69 100644 --- a/crates/sqlload/src/duckdb_source/main/read_catalog.rs +++ b/crates/sqlload/src/duckdb_source/main/read_catalog.rs @@ -62,6 +62,7 @@ pub async fn read_catalog( estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, }); } Ok(tables) diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index b93fb0b65..cc8d7a079 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -55,6 +55,8 @@ use crate::number_text::main::memory_text::memory_text; use crate::project::models::SendMode; use crate::batch_plan::main::window_concurrency::window_concurrency; +use crate::bigquery_source::main::selected_rows_query::selected_rows_query; +use crate::bigquery_source::models::BigQueryReadRequest; use crate::files_source::models::BatchFooters; use crate::files_source::models::DataFile; use crate::files_source::models::{FilesRead, FilesTable, SourceFormat, TextChecks}; @@ -165,16 +167,15 @@ use chrono::DateTime; use std::collections::BTreeSet; use std::sync::Arc; -use crate::loader::_helpers::table_plan::{ - added_constants, is_direct, pulls_unloaded, selects_rows, -}; +use crate::loader::_helpers::table_plan::added_constants; +use crate::routing::models::Route; /// Whether `append` can write the batches straight into the table: not a logged table (its rows wait for a changed `ctx.state`), and a writer that can (on Snowflake, a route loading files into the table itself), and, for an unload the destination reads itself, one that [loads it directly](unloads_directly). fn writes_directly(table: &TablePlan, writer: &Writer) -> bool { table.mode == WriteMode::Append && table.logged.is_none() && writer.writes_directly() - && (!pulls_unloaded(table.route.as_ref()) || unloads_directly(writer)) + && (!table.route.as_ref().is_some_and(Route::pulls_unloaded) || unloads_directly(writer)) } /// Where the batches go, from the publisher's fields: `staging` when there is one, and `columns` as the staged batches carry them. @@ -1046,7 +1047,7 @@ impl Publisher { let Some(read) = &self.table.files else { return Ok(None); }; - if !is_direct(self.table.route.as_ref()) { + if !self.table.route.as_ref().is_some_and(Route::is_direct) { return Ok(None); } let columns = staged_columns(&self.table); @@ -1083,8 +1084,13 @@ impl Publisher { source: UnloadedSource<'_>, ) -> Result { let columns = staged_columns(&self.table); + let merge_key = match &self.table.mode { + WriteMode::Merge { unique_key, .. } => unique_key.as_slice(), + _ => &[], + }; let batch = UnloadedBatch { source, + merge_key, read: &self.table.read_columns, write_columns: &self.table.schema.write_columns, target: write_target( @@ -1115,12 +1121,12 @@ impl Publisher { part: ReadPart, ) -> Result<(), LoaderError> { let table = self.table.clone(); - if selects_rows(table.route.as_ref()) { - let query = selected_query(&table, part)?; + if table.route.as_ref().is_some_and(Route::selects_rows) { + let query = selected_query(reader, &table, part)?; self.load_unloaded(UnloadedSource::Query(&query)).await?; return Ok(()); } - if pulls_unloaded(table.route.as_ref()) { + if table.route.as_ref().is_some_and(Route::pulls_unloaded) { unload_source(reader, &table, part, async |files| { self.load_unloaded(UnloadedSource::Files(files)).await }) @@ -1359,7 +1365,8 @@ impl Publisher { return Ok(BatchFooters::default()); }; if source_files.encoding.record.is_some() { - let direct = self.table.files.is_some() && is_direct(self.table.route.as_ref()); + let direct = self.table.files.is_some() + && self.table.route.as_ref().is_some_and(Route::is_direct); return match (source_files.encoding.format, direct) { (SourceFormat::Csv, true) => Ok(BatchFooters { headers: batch_headers(folder, files, &source_files.encoding).await?, @@ -1385,8 +1392,10 @@ impl Publisher { (columns, footers) } SourceFormat::Csv => { - let (Some(read), true) = (&self.table.files, is_direct(self.table.route.as_ref())) - else { + let (Some(read), true) = ( + &self.table.files, + self.table.route.as_ref().is_some_and(Route::is_direct), + ) else { return Ok(BatchFooters::default()); }; let headers = batch_headers(folder, files, &source_files.encoding).await?; @@ -1408,8 +1417,10 @@ impl Publisher { } SourceFormat::Xlsx | SourceFormat::Xml => return Ok(BatchFooters::default()), SourceFormat::Json | SourceFormat::Ndjson => { - let (Some(read), true) = (&self.table.files, is_direct(self.table.route.as_ref())) - else { + let (Some(read), true) = ( + &self.table.files, + self.table.route.as_ref().is_some_and(Route::is_direct), + ) else { return Ok(BatchFooters::default()); }; let Some(found) = file_keys(&self.writer, spec, files).await? else { @@ -1769,7 +1780,7 @@ async fn feed( transform: transform.cloned(), ..ReadPart::default() }; - match pulls_unloaded(table.route.as_ref()) { + match table.route.as_ref().is_some_and(Route::pulls_unloaded) { true => publisher.pass_part(reader, part).await?, false => { read_source(reader, table, part, async |batch| { @@ -1873,7 +1884,8 @@ pub(crate) fn published_once(table: &TablePlan) -> &'static str { /// Whether the target loads the table's files itself: a files source's, or a warehouse source's unload. pub(crate) fn reads_files_itself(table: &TablePlan) -> bool { - is_direct(table.route.as_ref()) || pulls_unloaded(table.route.as_ref()) + table.route.as_ref().is_some_and(Route::is_direct) + || table.route.as_ref().is_some_and(Route::pulls_unloaded) } /// The table's spool, in its `staging_dir` or the system's temporary folder, once that has room for about what a commit holds (see [`held_estimate`]), where the source and the file system say; commits are cut at `commit_bytes`. @@ -1955,11 +1967,27 @@ pub(crate) fn publishes_once(table: &TablePlan) -> bool { ) && sql_adapter(table.spec.warehouse).publishes_runs_once() } -/// The query an unload of `part` of `table` would run, which a destination writing through the source's own connection runs itself. -fn selected_query(table: &TablePlan, part: ReadPart) -> Result { +/// The query that selects `part` of `table` from its source, which a destination writing through the source's own connection runs itself: a Snowflake source's unload query, or the query a BigQuery source's rows would be read with. +fn selected_query( + reader: &SourceReader, + table: &TablePlan, + part: ReadPart, +) -> Result { let filter = table .spec .read_filter(part.filter.or_else(|| table.source_filter.clone())); - select_query(&unload_request(table, filter.as_deref())) - .map_err(|error| LoaderError::Invalid(error.to_string())) + match reader { + SourceReader::BigQuery(reader) => Ok(selected_rows_query( + reader, + &BigQueryReadRequest { + table: &table.spec.source_table, + sql: table.spec.query_sql(), + columns: &table.read_columns, + filter: filter.as_deref(), + filtered: table.spec.row_filter.is_some(), + }, + )), + _ => select_query(&unload_request(table, filter.as_deref())) + .map_err(|error| LoaderError::Invalid(error.to_string())), + } } diff --git a/crates/sqlload/src/loader/_helpers/python.rs b/crates/sqlload/src/loader/_helpers/python.rs index 597f050ff..88f49784f 100644 --- a/crates/sqlload/src/loader/_helpers/python.rs +++ b/crates/sqlload/src/loader/_helpers/python.rs @@ -154,6 +154,7 @@ pub(crate) fn python_tables(source: &SourceConfig) -> Vec { estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, }); } tables diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index 648fd23bc..ae44a9ff5 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -29,9 +29,7 @@ use crate::loader::_helpers::python::{list_choices, unnested_table}; use crate::loader::_helpers::resolve::display_name; use crate::loader::_helpers::run_log::recorded_commits; use crate::loader::_helpers::sources::{readers_note, unarchived}; -use crate::loader::_helpers::table_plan::{ - archived_reason, is_direct, pulls_unloaded, selects_rows, -}; +use crate::loader::_helpers::table_plan::archived_reason; use crate::loader::constants::{ CONSTANT_NOTE, DRAINED_REASON, LABEL_WIDTH, NULL_KEYS_NOTE, PLAN_DETAIL_COLUMN, PRIMARY_KEY_PLACEHOLDER, RESULT_SCHEMA_VERSION, SELECTED_ROUTE, @@ -678,9 +676,10 @@ pub(crate) fn load_item(table: &TablePlan) -> PlanItem { "changed columns move to the end of the column order", )); } - let staged_why = - (table.stage.is_some() && spec.files.is_some() && !is_direct(table.route.as_ref())) - || spec.warehouse_source.is_some(); + let staged_why = (table.stage.is_some() + && spec.files.is_some() + && !table.route.as_ref().is_some_and(Route::is_direct)) + || spec.warehouse_source.is_some(); let archived = archived_reason(spec, table.files.as_ref()).filter(|_| !staged_why); let why_shown = staged_why || table.route.as_ref().is_some_and(|route| { @@ -703,7 +702,7 @@ pub(crate) fn load_item(table: &TablePlan) -> PlanItem { route: WarehouseRoute::Unload { stage, .. }, .. }) = &spec.warehouse_source - && !selects_rows(table.route.as_ref()) + && !table.route.as_ref().is_some_and(Route::selects_rows) { details.push(detail("stage", stage.as_str())); } @@ -716,7 +715,7 @@ pub(crate) fn load_item(table: &TablePlan) -> PlanItem { details.push(detail("why", reason)); } if table.route.as_ref().is_some_and(|route| { - is_direct(Some(route)) + route.is_direct() && spec .masked .columns @@ -844,7 +843,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { thousands(MAX_BATCH_FILES as u64) ) } - (_, true) if pulls_unloaded(table.route.as_ref()) => { + (_, true) if table.route.as_ref().is_some_and(Route::pulls_unloaded) => { "every row loaded at once into a new table, swapped in once at the end".to_owned() } (_, true) if sql_adapter(spec.warehouse).keeps_tables() => { @@ -866,7 +865,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { _ => "", } ), - (CommitRule::Size, false) if pulls_unloaded(table.route.as_ref()) => { + (CommitRule::Size, false) if table.route.as_ref().is_some_and(Route::pulls_unloaded) => { "one commit: the destination loads every row at once".to_owned() } (CommitRule::Size, false) => "about every 1 GB".to_owned(), diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index c20413ecd..bdae4d7fe 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -714,6 +714,7 @@ async fn sheets_tables( estimated_bytes: 0, comment: None, listed_keys: Vec::new(), + other_location: None, }); } match problems.is_empty() { diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index 3667c5204..36b144ef0 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -554,7 +554,11 @@ pub(crate) fn build_spec( _ => MOTHERDUCK_SOURCE, }, sql: overrides.sql.clone(), - route: WarehouseRoute::Query, + route: WarehouseRoute::Query { + same_connection: kind == SourceKind::BigQuery + && context.target.session_connection.as_ref() + == Some(&context.source.connection), + }, }), _ => warehouse_source(context, overrides), }, @@ -1513,7 +1517,11 @@ fn nesting( Unnest::None } Some(PathsSetting::Paths(paths)) => { - problems.extend(path_problems(paths, max_depth)); + problems.extend( + paths + .iter() + .filter_map(|path| path_problem(path, paths, max_depth)), + ); problems.extend(column_setting_problems(context, overrides, paths)); Unnest::Paths(paths.clone()) } @@ -1574,14 +1582,6 @@ pub(crate) fn table_compat( .unwrap_or_default() } -/// The `unnest` paths with an empty part, a parent path not also listed, or more parts than `max_depth`. -fn path_problems(paths: &[String], max_depth: u32) -> Vec { - paths - .iter() - .filter_map(|path| path_problem(path, paths, max_depth)) - .collect() -} - /// What is wrong with the `unnest` path `path` among `paths`: an empty part, a parent path not also listed, or more parts than `max_depth`. fn path_problem(path: &str, paths: &[String], max_depth: u32) -> Option { let parts: Vec<&str> = path.split('.').collect(); diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 11cd8fa8d..97aa6c732 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -38,6 +38,7 @@ use crate::batch_plan::main::ignored_concurrency::ignored_concurrency; use crate::batch_plan::main::split_windows::split_windows; use crate::batch_plan::models::{FileGroup, WindowSplit}; use crate::bigquery_source::constants::BIGQUERY_SOURCE; +use crate::bigquery_source::main::select_problem::select_problem; use crate::cdc::constants::{CDC_PARTITION_PROBLEM, CDC_RELOAD_PROBLEM, PLAIN_JSON_TYPE}; use crate::cdc::main::metadata_read_columns::metadata_read_columns; use crate::column_types::main::is_variant::is_variant; @@ -449,7 +450,7 @@ pub(crate) fn read_columns(spec: &TableSpec) -> Result, Vec + .is_some_and(|source| matches!(source.route, WarehouseRoute::Query { .. })) => { crate::odbc_source::main::read_columns::read_columns( &selected_columns(spec), @@ -1076,22 +1077,38 @@ pub(crate) fn reader_offer( .. }) => vec![store.clone()], Some(WarehouseSource { - route: WarehouseRoute::Query, + route: WarehouseRoute::Query { .. }, .. }) | None => Vec::new(), }; let route = spec.warehouse_source.as_ref().map(|source| &source.route); + let unselected = match route { + Some(WarehouseRoute::Query { + same_connection: true, + }) => needs_loader + .clone() + .or_else(|| { + (!spec.masked.columns.is_empty()).then(|| "the table masks columns".to_owned()) + }) + .or_else(|| select_problem(&selected_columns(spec), &spec.pinned)), + _ => None, + }; let selected_by_destination = spec.masked.columns.is_empty() && matches!( route, - Some(WarehouseRoute::Unload { - same_connection: true, - .. - }) - ); + Some( + WarehouseRoute::Unload { + same_connection: true, + .. + } | WarehouseRoute::Query { + same_connection: true + } + ) + ) + && unselected.is_none(); let direct_blocked = match unload_to.is_empty() { - true => direct_blocked, + true => direct_blocked.or(unselected), false => unload_blocked( spec.warehouse, (!spec.masked.columns.is_empty(), &spec.source_table.columns), @@ -1131,42 +1148,6 @@ fn one_file_each(count: usize) -> Vec { .collect() } -/// Whether `route` has the destination select a warehouse source's rows itself, inside the warehouse. -pub(crate) fn selects_rows(route: Option<&Route>) -> bool { - route.is_some_and(|route| matches!(route.steps.as_slice(), [Step::DestinationSelects])) -} - -/// Whether `route` has the destination load a warehouse source's rows itself: its unloaded files, or the rows it selects inside the warehouse. -pub(crate) fn pulls_unloaded(route: Option<&Route>) -> bool { - route.is_some_and(|route| { - matches!( - route.steps.as_slice(), - [Step::Unload(_), Step::DestinationPulls(_)] - | [Step::DestinationSelects] - | [ - Step::Unload(_), - Step::CopyFiles(Store::Local(_)), - Step::DestinationPulls(_) - ] - ) - }) -} - -/// Whether `route` has the destination load a files source's own files. -pub(crate) fn is_direct(route: Option<&Route>) -> bool { - route.is_some_and(|route| { - matches!( - route.steps.as_slice(), - [Step::UseFiles(_), Step::DestinationPulls(_)] - | [ - Step::UseFiles(_), - Step::CopyFiles(_), - Step::DestinationPulls(_) - ] - ) - }) -} - /// A warning for each `hmac`-masked column of a files source whose files `destination` would otherwise read itself: every file then goes through SQLLoad. fn hmac_warnings( spec: &TableSpec, diff --git a/crates/sqlload/src/loader/constants.rs b/crates/sqlload/src/loader/constants.rs index 09865f007..68b0b7cad 100644 --- a/crates/sqlload/src/loader/constants.rs +++ b/crates/sqlload/src/loader/constants.rs @@ -16,7 +16,7 @@ pub const PLAN_DETAIL_COLUMN: usize = 52; /// The width of a plan detail's label and the space after it, as `route `. pub const LABEL_WIDTH: usize = 8; -/// The route of a table a Snowflake target selects from a source on its own connection, after the warehouse's name. +/// The route of a table a Snowflake or BigQuery target selects from a source on its own connection, after the warehouse's name. pub const SELECTED_ROUTE: &str = "INSERT ... SELECT on the source's connection (nothing through the loader)"; diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index 8ff5235ef..bab8c3bd4 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -333,8 +333,8 @@ pub enum WarehouseRoute { stage: String, same_connection: bool, }, - /// SQLLoad runs the query and reads its rows as Arrow. - Query, + /// SQLLoad runs the query and reads its rows as Arrow. `same_connection`: the target writes through the source's own connection, so it can select the rows itself instead. + Query { same_connection: bool }, } /// A created table's layout, with SQLBuild's ClickHouse field names; every field unset by default. diff --git a/crates/sqlload/src/loader/tests/helpers.rs b/crates/sqlload/src/loader/tests/helpers.rs index 44003fe9d..3ac0492ec 100644 --- a/crates/sqlload/src/loader/tests/helpers.rs +++ b/crates/sqlload/src/loader/tests/helpers.rs @@ -22,7 +22,7 @@ use crate::loader::models::{FilesSource, PlanDetail, PlanItem, TableSpec, TableT use crate::loader::tests::test_types::{ BigQueryReaderTestCase, ChildPathsTestCase, DeclaredProblemsTestCase, GcsReaderTestCase, InferredTableTestCase, ItemSpec, OlderFilesTestCase, ReloadOrderTestCase, - SourceSpellingTestCase, TransformReaderTestCase, + SelectedReaderTestCase, SourceSpellingTestCase, TransformReaderTestCase, }; use crate::postgres_source::models::{PgType, SourceColumn, SourceTable}; use crate::project::main::parse_python_project::parse_python_project; @@ -56,6 +56,7 @@ pub(crate) fn inferred_columns(test_case: &InferredTableTestCase) -> String { estimated_bytes: -1, split_pages: None, listed_keys: Vec::new(), + other_location: None, }; let live: Vec = test_case .live @@ -432,6 +433,7 @@ pub(crate) fn refused_bounds_runs( split_pages: None, comment: None, listed_keys: Vec::new(), + other_location: None, }; let resolved = resolve_tables(&project, "erp", &[table]); let spec = resolved @@ -529,6 +531,7 @@ pub(crate) fn not_null_keys( split_pages: None, comment: None, listed_keys: Vec::new(), + other_location: None, }; let resolved = resolve_tables(&project, "erp", &[table]); resolved @@ -557,6 +560,32 @@ pub(crate) fn transform_reader( Ok(reader_offer(&spec, None, None).needs_loader) } +/// Whether BigQuery selects a BigQuery source's table itself through the source's own connection, with the case's mask and parse, and why SQLLoad reads it otherwise. +pub(crate) fn selected_reader( + test_case: &SelectedReaderTestCase, +) -> Result<(bool, Option), String> { + let mut spec = python_spec("{}")?; + spec.python = None; + spec.warehouse = crate::column_types::models::Warehouse::BigQuery; + spec.warehouse_source = Some(crate::loader::models::WarehouseSource { + system: crate::bigquery_source::constants::BIGQUERY_SOURCE, + sql: None, + route: crate::loader::models::WarehouseRoute::Query { + same_connection: true, + }, + }); + spec.masked.columns.extend( + test_case + .mask + .map(|(column, method)| (column.to_owned(), method)), + ); + let parse = column_parse(&serde_json::json!({ "timestamp": ["iso"] }))?; + spec.parses + .extend(test_case.parse.map(|column| (column.to_owned(), parse))); + let offer = reader_offer(&spec, None, None); + Ok((offer.selected_by_destination, offer.direct_blocked)) +} + /// Why DuckDB can't read a Parquet files table's files in GCS itself, with the case's HMAC key and endpoint on the connection. pub(crate) fn gcs_reader(test_case: &GcsReaderTestCase) -> Result, String> { let spec = files_spec(StoreKind::Gcs { @@ -741,6 +770,7 @@ pub(crate) fn unload_spec(url: &str) -> Result { split_pages: None, comment: None, listed_keys: Vec::new(), + other_location: None, }; let resolved = resolve_tables(&project, "marts", &[table]); resolved diff --git a/crates/sqlload/src/loader/tests/test_transform_reader.rs b/crates/sqlload/src/loader/tests/test_transform_reader.rs index a70a61912..4191ce6aa 100644 --- a/crates/sqlload/src/loader/tests/test_transform_reader.rs +++ b/crates/sqlload/src/loader/tests/test_transform_reader.rs @@ -1,8 +1,11 @@ use crate::column_types::models::ColumnType; -use crate::loader::tests::helpers::{bigquery_reader, gcs_reader, transform_reader}; +use crate::loader::tests::helpers::{ + bigquery_reader, gcs_reader, selected_reader, transform_reader, +}; use crate::loader::tests::test_types::{ - BigQueryReaderTestCase, GcsReaderTestCase, TransformReaderTestCase, + BigQueryReaderTestCase, GcsReaderTestCase, SelectedReaderTestCase, TransformReaderTestCase, }; +use crate::project::models::MaskMethod; #[test] fn given_a_files_table_with_a_transform_or_a_parse_when_routing_then_sqlload_reads_the_files_itself() @@ -172,3 +175,48 @@ fn given_parquet_files_in_gcs_when_routing_into_bigquery_then_bigquery_reads_the } Ok(()) } + +#[test] +fn given_a_bigquery_source_on_the_targets_connection_when_routing_then_bigquery_selects_the_rows_unless_sqlload_must_see_them_and_plan_says_why() +-> Result<(), String> { + let test_cases = [ + SelectedReaderTestCase { + description: "a plain table is selected by BigQuery itself", + mask: None, + parse: None, + expected_selected: true, + expected_reason: None, + }, + SelectedReaderTestCase { + description: "a hashed column is masked by SQLLoad, which says so", + mask: Some(("note", MaskMethod::Hash)), + parse: None, + expected_selected: false, + expected_reason: Some("the table masks columns"), + }, + SelectedReaderTestCase { + description: "an HMAC key must never appear in SQL", + mask: Some(("note", MaskMethod::Hmac)), + parse: None, + expected_selected: false, + expected_reason: Some("note masked with hmac, whose key must never appear in SQL"), + }, + SelectedReaderTestCase { + description: "a parse runs in SQLLoad, which says so", + mask: None, + parse: Some("updated"), + expected_selected: false, + expected_reason: Some("parse runs on updated"), + }, + ]; + for test_case in test_cases { + let (selected, reason) = selected_reader(&test_case)?; + assert_eq!( + (selected, reason.as_deref()), + (test_case.expected_selected, test_case.expected_reason), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs index 837dacc37..be5764205 100644 --- a/crates/sqlload/src/loader/tests/test_types.rs +++ b/crates/sqlload/src/loader/tests/test_types.rs @@ -285,3 +285,16 @@ pub(crate) struct BigQueryReaderTestCase { /// Why SQLLoad must read the files instead of BigQuery, if it must. pub(crate) expected_reason: Option<&'static str>, } + +/// A BigQuery source's table loading into BigQuery through the source's own connection. +pub(crate) struct SelectedReaderTestCase { + pub(crate) description: &'static str, + /// A column the table masks, with its method. + pub(crate) mask: Option<(&'static str, crate::project::models::MaskMethod)>, + /// A column the table parses. + pub(crate) parse: Option<&'static str>, + /// Whether BigQuery selects the rows itself. + pub(crate) expected_selected: bool, + /// Why SQLLoad reads the rows instead, as `plan` shows it. + pub(crate) expected_reason: Option<&'static str>, +} diff --git a/crates/sqlload/src/odbc_source/_helpers/catalog.rs b/crates/sqlload/src/odbc_source/_helpers/catalog.rs index 5448793ec..4afc28587 100644 --- a/crates/sqlload/src/odbc_source/_helpers/catalog.rs +++ b/crates/sqlload/src/odbc_source/_helpers/catalog.rs @@ -76,6 +76,7 @@ pub(crate) fn source_table( estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, } } @@ -132,6 +133,7 @@ pub(crate) fn catalog_tables( }; tables.push(SourceTable { listed_keys, + other_location: None, ..source_table((schema, table), columns, key) }); } diff --git a/crates/sqlload/src/odbc_source/tests/helpers.rs b/crates/sqlload/src/odbc_source/tests/helpers.rs index 02033d36b..8e863df9f 100644 --- a/crates/sqlload/src/odbc_source/tests/helpers.rs +++ b/crates/sqlload/src/odbc_source/tests/helpers.rs @@ -25,6 +25,7 @@ pub(crate) fn table(schema: &str, name: &str, columns: &[&str]) -> SourceTable { split_pages: None, comment: None, listed_keys: Vec::new(), + other_location: None, } } diff --git a/crates/sqlload/src/postgres_source/_helpers/catalog.rs b/crates/sqlload/src/postgres_source/_helpers/catalog.rs index 21dbed7c1..c6ba01926 100644 --- a/crates/sqlload/src/postgres_source/_helpers/catalog.rs +++ b/crates/sqlload/src/postgres_source/_helpers/catalog.rs @@ -77,6 +77,7 @@ pub(crate) fn tables_from_rows( split_pages: row.get(13), comment: row.get(12), listed_keys, + other_location: None, } }); table.columns.push(column_from_row(row)); diff --git a/crates/sqlload/src/postgres_source/models.rs b/crates/sqlload/src/postgres_source/models.rs index f59485121..d267043f0 100644 --- a/crates/sqlload/src/postgres_source/models.rs +++ b/crates/sqlload/src/postgres_source/models.rs @@ -39,6 +39,8 @@ pub struct SourceTable { pub comment: Option, /// The primary key and unique constraints the database lists for a table whose driver reported no primary key (ODBC, from `INFORMATION_SCHEMA`), which point at the driver's key lookup; empty otherwise. pub listed_keys: Vec, + /// A BigQuery table's location when query jobs on its connection run in another: the Storage Read API reads it from anywhere, but a query job can't; `None` otherwise. + pub other_location: Option, } /// A key constraint the database lists: `PRIMARY KEY` or `UNIQUE`, and its columns in order. @@ -62,6 +64,7 @@ impl SourceTable { estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, } } diff --git a/crates/sqlload/src/routing/_helpers/warehouse_source.rs b/crates/sqlload/src/routing/_helpers/warehouse_source.rs index de448edb8..4bce6495e 100644 --- a/crates/sqlload/src/routing/_helpers/warehouse_source.rs +++ b/crates/sqlload/src/routing/_helpers/warehouse_source.rs @@ -23,18 +23,12 @@ pub(crate) fn should_unload(reader: &ReaderOffer, read_via: ReadVia) -> Result Result { - if reader.selected_by_destination && reader.needs_loader.is_none() { - return Ok(Route { - steps: vec![Step::DestinationSelects], - reason: "the destination writes through the source's own connection, so it selects the rows itself; no data leaves the warehouse".to_owned(), - }); - } let stores = ordered_stores(&reader.unload_to, overrides.stage)?; let unblocked = reader.direct_blocked.is_none() && reader.needs_loader.is_none(); if let Some(store) = stores diff --git a/crates/sqlload/src/routing/main/plan_route.rs b/crates/sqlload/src/routing/main/plan_route.rs index 46f350cec..e1fe09733 100644 --- a/crates/sqlload/src/routing/main/plan_route.rs +++ b/crates/sqlload/src/routing/main/plan_route.rs @@ -7,7 +7,7 @@ use crate::routing::constants::LOADER_READS_BECAUSE; use crate::routing::errors::RouteError; use crate::routing::models::{ReaderOffer, Route, RouteOverrides, Step, WriterAccepts}; -/// The cheapest route from `reader` to `writer`, honouring `overrides`. +/// The cheapest route from `reader` to `writer`, honouring `overrides`: a warehouse source's rows selected by a destination writing through its own connection, unless the table needs the loader. pub fn plan_route( reader: &ReaderOffer, writer: &WriterAccepts, @@ -25,6 +25,12 @@ pub fn plan_route( (None, _) => other_reads(reader, writer, files, overrides), }; } + if reader.selected_by_destination && reader.needs_loader.is_none() { + return Ok(Route { + steps: vec![Step::DestinationSelects], + reason: "the destination writes through the source's own connection, so it selects the rows itself; no data leaves the warehouse".to_owned(), + }); + } if should_unload(reader, overrides.read_via)? { return unload_route(reader, writer, overrides); } @@ -33,10 +39,11 @@ pub fn plan_route( "it neither streams rows nor has files".to_owned(), )); } - deliver_arrow( - vec![Step::ReadArrow], - writer, - overrides.stage, - "the source streams rows", - ) + let reason = match &reader.direct_blocked { + Some(reason) => format!( + "the source streams rows, as the destination can't select them itself: {reason}" + ), + None => "the source streams rows".to_owned(), + }; + deliver_arrow(vec![Step::ReadArrow], writer, overrides.stage, &reason) } diff --git a/crates/sqlload/src/routing/models.rs b/crates/sqlload/src/routing/models.rs index c8310c402..d58faccd1 100644 --- a/crates/sqlload/src/routing/models.rs +++ b/crates/sqlload/src/routing/models.rs @@ -191,3 +191,37 @@ pub struct Route { pub steps: Vec, pub reason: String, } + +impl Route { + /// Whether the route has the destination select a warehouse source's rows itself, inside the warehouse. + pub fn selects_rows(&self) -> bool { + matches!(self.steps.as_slice(), [Step::DestinationSelects]) + } + + /// Whether the route has the destination load a warehouse source's rows itself: its unloaded files, or the rows it selects inside the warehouse. + pub fn pulls_unloaded(&self) -> bool { + matches!( + self.steps.as_slice(), + [Step::Unload(_), Step::DestinationPulls(_)] + | [Step::DestinationSelects] + | [ + Step::Unload(_), + Step::CopyFiles(Store::Local(_)), + Step::DestinationPulls(_) + ] + ) + } + + /// Whether the route has the destination load a files source's own files. + pub fn is_direct(&self) -> bool { + matches!( + self.steps.as_slice(), + [Step::UseFiles(_), Step::DestinationPulls(_)] + | [ + Step::UseFiles(_), + Step::CopyFiles(_), + Step::DestinationPulls(_) + ] + ) + } +} diff --git a/crates/sqlload/src/routing/tests/test_plan_route.rs b/crates/sqlload/src/routing/tests/test_plan_route.rs index 2c6dd8755..74e50abc4 100644 --- a/crates/sqlload/src/routing/tests/test_plan_route.rs +++ b/crates/sqlload/src/routing/tests/test_plan_route.rs @@ -227,6 +227,29 @@ fn given_a_source_and_destination_when_planning_then_the_cheapest_route_is_chose overrides: RouteOverrides::default(), expected_steps: &["destination selects the rows itself"], }, + PlanRouteTestCase { + description: "a source that streams Arrow and never unloads is selected by a destination writing through its own connection", + reader: ReaderOffer { + arrow: true, + selected_by_destination: true, + ..ReaderOffer::default() + }, + writer: database_destination(), + overrides: RouteOverrides::default(), + expected_steps: &["destination selects the rows itself"], + }, + PlanRouteTestCase { + description: "a source that streams Arrow on the destination's connection still streams a table that needs the loader", + reader: ReaderOffer { + arrow: true, + selected_by_destination: true, + needs_loader: Some("the table has checks".to_owned()), + ..ReaderOffer::default() + }, + writer: database_destination(), + overrides: RouteOverrides::default(), + expected_steps: &["read as Arrow", "destination takes Arrow"], + }, PlanRouteTestCase { description: "on the source's own connection, a table that needs the loader is still unloaded and read back", reader: ReaderOffer { diff --git a/crates/sqlload/src/snowflake_source/_helpers/catalog.rs b/crates/sqlload/src/snowflake_source/_helpers/catalog.rs index b3b3b57d9..1ca6e3cfe 100644 --- a/crates/sqlload/src/snowflake_source/_helpers/catalog.rs +++ b/crates/sqlload/src/snowflake_source/_helpers/catalog.rs @@ -71,6 +71,7 @@ pub(crate) fn catalog_tables( estimated_bytes: number(9).unwrap_or(-1), comment: cell(10).filter(|comment| !comment.trim().is_empty()), listed_keys: Vec::new(), + other_location: None, }); table.columns.push(source_column( &text(2), @@ -119,6 +120,7 @@ pub(crate) fn listed_tables( estimated_bytes: number("bytes"), comment: Some(text("comment")).filter(|comment| !comment.trim().is_empty()), listed_keys: Vec::new(), + other_location: None, }; tables.insert((schema, name), table); } diff --git a/crates/sqlload/src/snowflake_source/main/read_catalog.rs b/crates/sqlload/src/snowflake_source/main/read_catalog.rs index fa495449f..cb1701163 100644 --- a/crates/sqlload/src/snowflake_source/main/read_catalog.rs +++ b/crates/sqlload/src/snowflake_source/main/read_catalog.rs @@ -102,5 +102,6 @@ async fn query_table( estimated_bytes: -1, comment: None, listed_keys: Vec::new(), + other_location: None, }) } diff --git a/crates/sqlload/src/snowflake_source/tests/helpers.rs b/crates/sqlload/src/snowflake_source/tests/helpers.rs index 7e2997e44..e1774301c 100644 --- a/crates/sqlload/src/snowflake_source/tests/helpers.rs +++ b/crates/sqlload/src/snowflake_source/tests/helpers.rs @@ -53,6 +53,7 @@ pub(crate) fn events() -> SourceTable { split_pages: None, comment: None, listed_keys: Vec::new(), + other_location: None, } } diff --git a/website/public/index.html b/website/public/index.html index 91b1d7198..75d491c87 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -173,7 +173,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, Parquet in S3 or Google Cloud Storage into MotherDuck on its servers, and Parquet in Google Cloud Storage into BigQuery as a temporary external table. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, Parquet in S3 or Google Cloud Storage into MotherDuck on its servers, and Parquet in Google Cloud Storage into BigQuery as a temporary external table. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake, and BigQuery tables into BigQuery on the same connection. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
diff --git a/website/src/content/docs/docs/sources/bigquery.mdx b/website/src/content/docs/docs/sources/bigquery.mdx
index c320bee6d..ec1096f31 100644
--- a/website/src/content/docs/docs/sources/bigquery.mdx
+++ b/website/src/content/docs/docs/sources/bigquery.mdx
@@ -90,6 +90,31 @@ the table's description.
   runs.
 - **Query tables** run as a query job too, and also read the data as it is when the job runs.
 
+## BigQuery loads the rows itself
+
+When the target is BigQuery and writes through the source's own `connection`, no rows pass through
+SQLLoad. BigQuery selects each batch's rows with one `INSERT ... SELECT` query job into the load's
+staging or build table, which then publishes as any other load does. A table is read as of the
+moment its load starts, as a read session reads it, and views, query tables and tables with `where`
+run their query in the same job. The rows load exactly as SQLLoad's reader would load them, with
+the same types.
+
+`plan` shows the route:
+
+```text
+route   BigQuery INSERT ... SELECT on the source's connection (nothing through the loader)
+```
+
+SQLLoad still reads the rows itself, and `plan -v` on the table says why, for a table that:
+
+- has a `STRUCT`, `RANGE`, `GEOGRAPHY` or repeated column, which SQLLoad converts to JSON or text;
+- pins a column to another type than its own;
+- is in another location than the connection's, which a read session reaches but a query job
+  doesn't;
+- has a `mask`, `parse`, `transform` or checks, which SQLLoad applies to the rows.
+
+A target on another `bigquery` connection, even with the same key, reads through SQLLoad as before.
+
 ## Query tables
 
 A table entry with `sql` is a query table: its rows are the query's result, and the entry's key is
@@ -158,7 +183,7 @@ fraction in a pinned integer, fails the read too. NaN and infinities inside a re
 ## Cost
 
 Read sessions bill the bytes of the columns and rows they read; the first 300 TB a month are free.
-Views and query tables also bill their query job. Listing tables, describing them and dry runs
+Views and query tables also bill their query job. A table [BigQuery loads itself](#bigquery-loads-the-rows-itself) bills its query job's scan instead of a read session. Listing tables, describing them and dry runs
 bill nothing.
 
 A table read from a cursor with `null_cursor = "fail"` (the default) and a cursor column
diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx
index 8bef8fc78..7588a1f55 100644
--- a/website/src/content/docs/docs/supported-systems.mdx
+++ b/website/src/content/docs/docs/supported-systems.mdx
@@ -11,7 +11,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). |
 | Snowflake | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or by DuckDB, Databricks or ClickHouse themselves from an S3 stage. A Snowflake target selects or copies the rows inside Snowflake when it writes through the source's connection, or its external stage covers the unload; see [Snowflake loads the rows itself](/docs/sources/snowflake/#snowflake-loads-the-rows-itself). See [Snowflake](/docs/sources/snowflake/). |
 | Databases with an ODBC driver: SQL Server, MySQL, MariaDB, OpenEdge and others | Tables and views, by `include` patterns, and query results | The driver, as Arrow. Tested with psqlODBC, Microsoft ODBC Driver 18 for SQL Server and MySQL Connector/ODBC; MariaDB Connector/ODBC and OpenEdge are checked by hand. See [ODBC](/docs/sources/odbc/#drivers). |
-| BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). |
+| BigQuery (beta) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. A BigQuery target on the source's connection selects the rows itself; see [BigQuery loads the rows itself](/docs/sources/bigquery/#bigquery-loads-the-rows-itself). See [BigQuery](/docs/sources/bigquery/). |
 | Databricks (beta) | Tables and views, by `include` patterns, and query results | One `SELECT` on the SQL warehouse, its result downloaded as Arrow files in parallel through the Statement Execution API's links. See [Databricks](/docs/sources/databricks/). |
 | DuckDB | Tables and views of a database file, by `include` patterns, and query results | One `SELECT` on a read-only connection, streamed as Arrow. See [DuckDB](/docs/sources/duckdb/). |
 | MotherDuck (beta) | Tables and views of one database, by `include` patterns, and query results | One `SELECT` run by a separate DuckDB 1.5 process (`sqlload[motherduck]`), its Arrow batches handed over one at a time. See [MotherDuck](/docs/sources/motherduck/). |
@@ -47,7 +47,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes
 | Postgres changes (`postgres_cdc`) | Yes | Yes | No | No | No | Yes | Yes, as a changelog |
 | Snowflake | Yes | Yes | Yes | Yes | Yes | Yes | Yes |
 | ODBC | Yes, untested | Yes, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes |
-| BigQuery (beta) | Yes, untested | Yes, tested live | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested |
+| BigQuery (beta) | Yes, untested | Yes, tested live | Yes, tested live | Yes, untested | Yes, untested | Yes | Yes, untested |
 | Databricks (beta) | Yes, untested | Yes, untested | Yes, untested | Yes, tested live | Yes, tested live | Yes, tested live | Yes, tested live |
 | DuckDB | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes | Yes, untested |
 | MotherDuck (beta) | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, untested | Yes, tested live | Yes, untested |
@@ -75,7 +75,7 @@ them into DuckDB.
 | MotherDuck | Live against a MotherDuck account on DuckDB 1.5.6: every write strategy, an incremental load and rerun, a new column and every portable type. Offline, every change loads through the separate process into a stand-in for MotherDuck and compares with DuckDB. As a source, live and offline: tables, a query table, `where`, `exclude_columns` and a cursor merge into DuckDB. See [MotherDuck](/docs/targets/motherduck/#tested). |
 | ODBC | psqlODBC against real Postgres in containers, into DuckDB, plus a live round from SQL Server and MySQL into Databricks. |
 | Databricks source | A fake Statement Execution API in tests (catalog discovery, result files in several chunks, an expired link fetched again, cursor reads, no token on downloads), plus a live round from a real SQL warehouse into DuckDB, Parquet files, Postgres and ClickHouse: tables, a view, a windowed query table, merges and every column shape. See [Databricks](/docs/sources/databricks/#tested). |
-| BigQuery source | A fake Storage Read API in tests (parallel streams, a broken stream resumed, cursor filters), and live loads from a sandbox project into DuckDB: tables, views, query tables, and integer, timestamp, text, `NUMERIC` and `BIGNUMERIC` cursors, plus a live round into Databricks. |
+| BigQuery source | A fake Storage Read API in tests (parallel streams, a broken stream resumed, cursor filters), and live loads from a sandbox project into DuckDB: tables, views, query tables, and integer, timestamp, text, `NUMERIC` and `BIGNUMERIC` cursors, plus a live round into Databricks, and live loads into BigQuery on the same connection (append by cursor, merge, a view replaced) whose tables match SQLLoad's reader row for row. |
 | S3 | A real S3-compatible store in containers; benchmarks against AWS S3. |
 | Local files | Real files in temporary folders. |
 | SFTP | OpenSSH's SFTP server in a container, into DuckDB: new and rewritten files, gzip CSV and Parquet, key and password sign-in, and unknown or changed host keys refused, plus a live round into Databricks. |
diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx
index 9283798f0..a2a374b84 100644
--- a/website/src/content/docs/docs/targets/index.mdx
+++ b/website/src/content/docs/docs/targets/index.mdx
@@ -16,7 +16,7 @@ names the route for every table.
 | MotherDuck (beta) | In S3, or in GCS with an HMAC key: MotherDuck's servers read them with `read_parquet`. Otherwise read by SQLLoad, appended as Arrow | Read by SQLLoad, appended as Arrow | Appended as Arrow |
 | Snowflake | In S3: `COPY` from an external stage over the bucket. Otherwise read by SQLLoad, then loaded as rows are | In S3: `COPY` from an external stage over the bucket. Otherwise as for Parquet files | Written as Parquet to the `load_method`'s stage, then `COPY` (or streamed or inserted, if the `load_method` says so) |
 | Databricks (beta) | In S3: `COPY INTO` where they lie, through an external location. In a folder: staged by SQLLoad in the volume, then `COPY INTO` | Read by SQLLoad, staged as Parquet in the volume, then `COPY INTO` | Written as Parquet to the volume, then `COPY INTO` |
-| BigQuery (beta) | In GCS, on a table that excludes `_sqlload_source_row`: BigQuery queries them where they lie into a staging table. Otherwise read by SQLLoad, streamed as Arrow into a staging table, then one query job | As for Parquet files | Streamed as Arrow into a staging table, then one query job |
+| BigQuery (beta) | In GCS, on a table that excludes `_sqlload_source_row`: BigQuery queries them where they lie into a staging table. Otherwise read by SQLLoad, streamed as Arrow into a staging table, then one query job | As for Parquet files | Streamed as Arrow into a staging table, then one query job. A BigQuery source on the target's own connection: BigQuery selects the rows with `INSERT ... SELECT` |
 | ClickHouse | In S3: ClickHouse reads them with `s3()` into a staging table. In a folder: read by SQLLoad, sent as Arrow | Read by SQLLoad, sent as Arrow into a staging table | Sent as Arrow into a staging table, then swapped in by table or partition |
 | Postgres | Read by SQLLoad, then binary `COPY` into a temporary staging table | As for Parquet files | Binary `COPY` into a temporary staging table, then one transaction per commit |