Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub(crate) mod pull;
pub(crate) mod selected;
pub(crate) mod sql_adapter;
pub(crate) mod type_names;
pub(crate) mod write;
114 changes: 114 additions & 0 deletions crates/sqlload/src/adapters/bigquery/_helpers/dialect/selected.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
//! The statement that has BigQuery select a BigQuery source's rows itself.

use crate::adapters::bigquery::models::BigQuerySql;
use crate::adapters::contract::constants::UNLOADED_BATCH_ROWS;
use crate::adapters::contract::models::TableName;
use crate::adapters::contract::types::SqlAdapter;
use crate::column_types::models::ColumnType;
use crate::load_naming::main::metadata_role::metadata_role;
use crate::load_naming::models::{Compat, MetadataRole};
use crate::postgres_source::models::ReadColumn;
use crate::schema_plan::constants::MERGE_ROW_COLUMN;
use crate::schema_plan::models::DesiredColumn;

/// One batch of a BigQuery source's rows BigQuery selects itself, into the staging table or, when it `builds`, a build table in the final types.
#[derive(Debug, Clone, Copy)]
pub(crate) struct SelectedLoad<'a> {
pub into: &'a TableName,
pub columns: &'a [DesiredColumn],
/// The source's read columns, each with its quoted name in `query`'s result.
pub read: &'a [ReadColumn],
/// The table's `constant_columns`: name, type and value as text.
pub constants: &'a [(String, ColumnType, String)],
pub compat: Compat,
/// The batch, from 1: a merge's row number counts after earlier batches'.
pub batch: u64,
/// A merge's `unique_key`: its row number counts among the rows sharing a key, which are all its dedupe compares, so BigQuery numbers them in parallel rather than on one worker.
pub merge_key: &'a [String],
pub builds: bool,
pub load_id: &'a str,
pub loaded_at: &'a str,
/// The `SELECT` of the source's rows.
pub query: &'a str,
}

/// `INSERT INTO into` every one of `columns` from the rows of `query`: a read column by its name in the query's result, a constant from its value, a merge's row number after the earlier batches', among the rows sharing its key, grouped as the merge groups them, in the order the query gives them (a key column the query doesn't read is a constant, the same for every row), the load id and load time the batch's; any other column NULL. Each is cast to its staging or build type without parameters, which a cast can't name and the column then checks; JSON goes into a staging table as its text, as SQLLoad's reader stages it.
pub(crate) fn selected_insert(load: &SelectedLoad<'_>) -> String {
let literal = |text: &str| BigQuerySql.render_loader_value_literal(text);
let mut values: Vec<String> = Vec::with_capacity(load.columns.len());
for column in load.columns {
let constant = load
.constants
.iter()
.find(|(name, _, _)| *name == column.name)
.map(|(_, _, value)| (literal(value), true));
let read = load
.read
.iter()
.find(|read| read.name == column.name)
.map(|read| (format!("u.{}", read.expression), false));
let value =
match (
metadata_role(load.compat, &column.name),
column.name.as_str(),
) {
(Some(MetadataRole::LoadId), _) => (literal(load.load_id), true),
(Some(MetadataRole::LoadedAt), _) => (literal(load.loaded_at), true),
(None, MERGE_ROW_COLUMN) => {
let mut keys: Vec<String> = Vec::new();
for read in load.read {
if load.merge_key.contains(&read.name) {
keys.push(BigQuerySql.render_partition_key(
&format!("u.{}", read.expression),
&read.output,
));
}
}
let over = match keys.is_empty() {
true => String::new(),
false => format!("PARTITION BY {}", keys.join(", ")),
};
values.push(format!(
"{} * {UNLOADED_BATCH_ROWS} + ROW_NUMBER() OVER ({over}) - 1",
load.batch
));
continue;
}
_ => constant
.or(read)
.unwrap_or_else(|| ("NULL".to_owned(), true)),
};
values.push(typed(value, &column.column_type, load.builds));
}
let targets: Vec<String> = load
.columns
.iter()
.map(|column| BigQuerySql.render_identifier(&column.name))
.collect();
format!(
"INSERT INTO {} ({}) SELECT {} FROM ({}) AS u",
BigQuerySql.render_qualified_name(load.into),
targets.join(", "),
values.join(", "),
load.query
)
}

/// `value` as a column of `column_type` holds it in the staging table, or in a build table (`builds`): JSON as its text in staging and as JSON in a build table (parsed when the value is text, `from_text`: a literal or NULL), anything else cast to its type without parameters.
fn typed((value, from_text): (String, bool), column_type: &ColumnType, builds: bool) -> String {
match (column_type, builds, from_text) {
(ColumnType::Json, false, false) => format!("TO_JSON_STRING({value})"),
(ColumnType::Json, true, false) => value,
(ColumnType::Json, true, true) => format!("PARSE_JSON({value})"),
_ => {
let sql_type = match builds {
true => BigQuerySql.render_loader_logical_type(column_type).sql,
false => BigQuerySql.render_staging_type(column_type),
};
let base = sql_type
.split_once('(')
.map_or(sql_type.as_str(), |(base, _)| base);
format!("CAST({value} AS {base})")
}
}
}
50 changes: 8 additions & 42 deletions crates/sqlload/src/adapters/bigquery/_helpers/write/direct_load.rs
Original file line number Diff line number Diff line change
@@ -1,18 +1,15 @@
//! A files batch loaded by BigQuery querying the source's own Parquet files in GCS.

use serde_json::{Value, json};

use crate::adapters::bigquery::_helpers::dialect::pull::{
PulledFile, PulledGroup, pull_queries, unread_name,
};
use crate::adapters::bigquery::_helpers::write::insert_job::insert_job;
use crate::adapters::bigquery::main::plan::_direct_problem::direct_problem;
use crate::adapters::bigquery::models::BigQueryWriter;
use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem;
use crate::adapters::contract::main::_footer_rows::footer_rows;
use crate::adapters::contract::models::{DirectBatch, DirectLoaded, TableName};
use crate::adapters::contract::types::TableWriter;
use crate::bigquery_api::main::load_job::load_job;
use crate::bigquery_api::models::BigQueryClient;
use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN};
use crate::files_source::main::normal_value::normal_value;
use crate::files_source::models::{BatchFooters, DataFile};
Expand Down Expand Up @@ -66,7 +63,13 @@ impl BigQueryWriter {
);
let mut rows = 0;
for (query, definitions) in queries {
rows += query_files(&self.destination.api, query, definitions).await?;
rows += insert_job(
&self.destination.api,
&query,
Some(definitions),
"this batch's files from GCS",
)
.await?;
}
let expected = footer_rows(batch.footers);
if rows != expected {
Expand Down Expand Up @@ -163,40 +166,3 @@ fn footer_fields(footers: &BatchFooters, data: &DataFile) -> Vec<String> {
.map(|field| field.name().clone())
.collect()
}

/// Run one query job inserting files in GCS, its temporary external tables `definitions`: the rows it inserted. A job BigQuery fails, or whose count is not a number, fails the batch.
async fn query_files(
api: &BigQueryClient,
query: String,
definitions: Value,
) -> Result<u64, LoaderError> {
let job = json!({
"jobReference": {
"projectId": api.project,
"jobId": format!("sqlload_files_{}", uuid::Uuid::new_v4().simple()),
"location": api.location,
},
"configuration": {"query": {
"query": query,
"useLegacySql": false,
"tableDefinitions": definitions,
}},
});
let resource = load_job(api, &job, None).await?;
let error = &resource["status"]["errorResult"];
if !error.is_null() {
return Err(LoaderError::Invalid(format!(
"BigQuery could not load this batch's files from GCS: {}: {}",
error["reason"].as_str().unwrap_or_default(),
error["message"].as_str().unwrap_or_default()
)));
}
let affected = resource["statistics"]["query"]["numDmlAffectedRows"]
.as_str()
.unwrap_or_default();
affected.parse().map_err(|_| {
LoaderError::Invalid(format!(
"BigQuery's query job reported {affected:?} inserted rows, not a count; the batch is not committed"
))
})
}
45 changes: 45 additions & 0 deletions crates/sqlload/src/adapters/bigquery/_helpers/write/insert_job.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
//! One `INSERT ... SELECT` BigQuery runs as a query job, counting the rows it inserted.

use serde_json::{Value, json};

use crate::bigquery_api::main::load_job::load_job;
use crate::bigquery_api::models::BigQueryClient;
use crate::loader::errors::LoaderError;

/// Run `query` as one query job, with its temporary external tables `definitions` when it reads files: the rows it inserted. A job BigQuery fails, or whose count is not a number, fails the batch; `what` names what the job loads in the error.
pub(in crate::adapters::bigquery) async fn insert_job(
api: &BigQueryClient,
query: &str,
definitions: Option<Value>,
what: &str,
) -> Result<u64, LoaderError> {
let mut configuration = json!({"query": query, "useLegacySql": false});
if let Some(definitions) = definitions {
configuration["tableDefinitions"] = definitions;
}
let job = json!({
"jobReference": {
"projectId": api.project,
"jobId": format!("sqlload_insert_{}", uuid::Uuid::new_v4().simple()),
"location": api.location,
},
"configuration": {"query": configuration},
});
let resource = load_job(api, &job, None).await?;
let error = &resource["status"]["errorResult"];
if !error.is_null() {
return Err(LoaderError::Invalid(format!(
"BigQuery could not load {what}: {}: {}",
error["reason"].as_str().unwrap_or_default(),
error["message"].as_str().unwrap_or_default()
)));
}
let affected = resource["statistics"]["query"]["numDmlAffectedRows"]
.as_str()
.unwrap_or_default();
affected.parse().map_err(|_| {
LoaderError::Invalid(format!(
"BigQuery's query job reported {affected:?} inserted rows, not a count; the batch is not committed"
))
})
}
2 changes: 2 additions & 0 deletions crates/sqlload/src/adapters/bigquery/_helpers/write/mod.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
pub(crate) mod arrow_rows;
pub(crate) mod direct_load;
pub(crate) mod insert_job;
pub(crate) mod selected_load;
pub(crate) mod table_writer;
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
//! A BigQuery source's rows selected by BigQuery itself, on the connection the target loads through.

use crate::adapters::bigquery::_helpers::dialect::selected::{SelectedLoad, selected_insert};
use crate::adapters::bigquery::_helpers::write::insert_job::insert_job;
use crate::adapters::bigquery::models::BigQueryWriter;
use crate::adapters::contract::models::{DirectLoaded, TableName, UnloadedBatch, UnloadedSource};
use crate::adapters::contract::types::TableWriter;
use crate::loader::errors::LoaderError;

impl BigQueryWriter {
/// Load `batch` by having BigQuery select the source's rows into the staging or build table the load publishes from, in one query job. Only a query, which plan gives a BigQuery source on the target's own connection, is loaded this way: BigQuery never reads unloaded files.
pub(in crate::adapters) async fn pull_unloaded(
&mut self,
batch: &UnloadedBatch<'_>,
) -> Result<DirectLoaded, LoaderError> {
let UnloadedSource::Query(query) = batch.source else {
return Err(LoaderError::Invalid(
"BigQuery does not load unloaded files; the plan should have had SQLLoad read them"
.to_owned(),
));
};
self.settle_rows().await?;
let target = &batch.target;
let (into, columns) = match (target.staging, target.builds) {
(Some(staging), false) => (staging, target.columns),
(staging, _) => (staging.unwrap_or(target.table), batch.write_columns),
};
let into = TableName {
database: Some(
into.database
.clone()
.unwrap_or_else(|| self.destination.project.clone()),
),
..into.clone()
};
let insert = selected_insert(&SelectedLoad {
into: &into,
columns,
read: batch.read,
constants: target.constants,
compat: target.compat,
batch: target.batch,
merge_key: batch.merge_key,
builds: target.builds,
load_id: target.load_id,
loaded_at: batch.loaded_at,
query,
});
let rows = insert_job(&self.destination.api, &insert, None, "the source's rows").await?;
Ok(DirectLoaded::Staged(rows))
}
}
2 changes: 2 additions & 0 deletions crates/sqlload/src/adapters/bigquery/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,7 @@ mod dialect;
mod helpers;
#[path = "tests/test_pull_query.rs"]
mod pull_query;
#[path = "tests/test_selected_insert.rs"]
mod selected_insert;
#[path = "tests/test_types.rs"]
mod test_types;
Loading
Loading