Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ pub(crate) mod publish;
pub(crate) mod pull;
pub(crate) mod sql_adapter;
pub(crate) mod type_names;
pub(crate) mod unloaded;
pub(crate) mod write;
35 changes: 20 additions & 15 deletions crates/sqlload/src/adapters/clickhouse/_helpers/dialect/pull.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ pub(crate) fn pull_insert(
(file, credentials): (&PulledFile, Option<&PullCredentials>),
(compat, load_id, loaded_at): (Compat, &str, &str),
) -> String {
let literal = |text: &str| ClickHouseSql.render_loader_value_literal(text);
let identifier = |name: &str| ClickHouseSql.render_identifier(name);
let cast = |value: String, column: &DesiredColumn| {
format!("CAST({value} AS {})", sql_type(&column.column_type))
Expand Down Expand Up @@ -116,8 +115,26 @@ pub(crate) fn pull_insert(
.iter()
.map(|column| identifier(&column.name))
.collect();
let numbered = columns
.iter()
.any(|column| column.name == SOURCE_ROW_COLUMN);
let settings = match numbered {
true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"),
false => String::new(),
};
format!(
"INSERT INTO {into} ({}) SELECT {} FROM {}{settings}",
targets.join(", "),
values.join(", "),
s3_parquet(&file.url, credentials)
)
}

/// `s3()` reading the Parquet file at `url`, signed with `credentials` (named as [query parameters](credential_parameters)), or unsigned (`NOSIGN`) when there are none.
pub(crate) fn s3_parquet(url: &str, credentials: Option<&PullCredentials>) -> String {
let literal = |text: &str| ClickHouseSql.render_loader_value_literal(text);
let parameter = |name: &str| format!("{{{name}:String}}");
let mut signed = vec![literal(&file.url)];
let mut signed = vec![literal(url)];
match credentials {
Some(credentials) => {
signed.extend([parameter(KEY_ID_PARAMETER), parameter(SECRET_PARAMETER)]);
Expand All @@ -131,19 +148,7 @@ pub(crate) fn pull_insert(
None => signed.push("NOSIGN".to_owned()),
}
signed.push(literal("Parquet"));
let numbered = columns
.iter()
.any(|column| column.name == SOURCE_ROW_COLUMN);
let settings = match numbered {
true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"),
false => String::new(),
};
format!(
"INSERT INTO {into} ({}) SELECT {} FROM s3({}){settings}",
targets.join(", "),
values.join(", "),
signed.join(", ")
)
format!("s3({})", signed.join(", "))
}

/// The HTTP URL ClickHouse reads `key` of `bucket` at: under `endpoint` by path when there is one (or with the bucket already in it, `in_endpoint`, for virtual-hosted requests to a custom endpoint), otherwise on AWS, in `region` when known, by virtual host unless the bucket's name holds a dot, which AWS's certificate does not cover. Each path segment is percent-encoded, which ClickHouse decodes before signing.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
//! The statement that has ClickHouse read one file of a warehouse source's unload from S3 itself.

use crate::adapters::clickhouse::_helpers::dialect::pull::{
PullCredentials, SqlTypeOf, s3_parquet,
};
use crate::adapters::clickhouse::_helpers::dialect::write::typed_literal;
use crate::adapters::clickhouse::constants::ORDERED_READ_SETTINGS;
use crate::adapters::clickhouse::models::ClickHouseSql;
use crate::adapters::contract::constants::{UNLOADED_BATCH_ROWS, UNLOADED_FILE_ROWS};
use crate::adapters::contract::models::WriteTarget;
use crate::adapters::contract::types::SqlAdapter;
use crate::column_types::models::ColumnType;
use crate::load_naming::main::metadata_role::metadata_role;
use crate::load_naming::models::MetadataRole;
use crate::postgres_source::models::ReadColumn;
use crate::schema_plan::constants::MERGE_ROW_COLUMN;
use crate::schema_plan::models::DesiredColumn;

/// One file of an unload that ClickHouse reads: its URL, and its place among the unload's files, from 0, which a merge's row number counts by.
pub(crate) struct UnloadedFile<'a> {
pub url: &'a str,
pub index: usize,
}

/// `INSERT INTO into` every one of `columns` (each typed by `sql_type`) from one of a warehouse's unloaded Parquet `file`s, read by [`s3_parquet`] with `credentials`. A column of `read` comes from the file, turned back from whole microseconds where it is read as a timestamp (a time never loads this way); a column of `target`'s constants from its value; a merge's row number from the batch, the file and the row in it, read in stored order; the load id and load time (`loaded_at`) are the batch's; any other column is NULL.
pub(crate) fn unloaded_insert(
(into, columns, sql_type): (&str, &[DesiredColumn], SqlTypeOf<'_>),
target: &WriteTarget<'_>,
(read, loaded_at): (&[ReadColumn], &str),
(file, credentials): (&UnloadedFile<'_>, Option<&PullCredentials>),
) -> String {
let values: Vec<String> = columns
.iter()
.map(|column| {
format!(
"CAST({} AS {})",
column_value(column, target, (read, loaded_at), file),
sql_type(&column.column_type)
)
})
.collect();
let targets: Vec<String> = columns
.iter()
.map(|column| ClickHouseSql.render_identifier(&column.name))
.collect();
let settings = match columns.iter().any(|column| column.name == MERGE_ROW_COLUMN) {
true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"),
false => String::new(),
};
format!(
"INSERT INTO {into} ({}) SELECT {} FROM {}{settings}",
targets.join(", "),
values.join(", "),
s3_parquet(file.url, credentials)
)
}

/// Where `column`'s value comes from, before its cast: the load id or load time, a merge's row number, a constant's value, the file's column of `read` it is, or NULL when the file lacks it.
fn column_value(
column: &DesiredColumn,
target: &WriteTarget<'_>,
(read, loaded_at): (&[ReadColumn], &str),
file: &UnloadedFile<'_>,
) -> String {
let constant = target
.constants
.iter()
.find(|(name, _, _)| *name == column.name);
let held = read.iter().find(|read| read.name == column.name);
match (metadata_role(target.compat, &column.name), constant, held) {
(Some(MetadataRole::LoadId), _, _) => typed_literal(target.load_id, &column.column_type),
(Some(MetadataRole::LoadedAt), _, _) => typed_literal(loaded_at, &column.column_type),
_ if column.name == MERGE_ROW_COLUMN => format!(
"{} * {UNLOADED_BATCH_ROWS} + {} * {UNLOADED_FILE_ROWS} + rowNumberInAllBlocks()",
target.batch, file.index
),
(_, Some((_, column_type, value)), _) => typed_literal(value, column_type),
(_, None, Some(read)) => {
unloaded_value(&read.output, &ClickHouseSql.render_identifier(&read.name))
}
(_, None, None) => "NULL".to_owned(),
}
}

/// File column `quoted` as the value it was unloaded from: a timestamp, with or without time zone, from its whole microseconds as the UTC instant SQLLoad's reader gives (ClickHouse stores both as an instant); a binary through `substring`, which keeps a fixed-size value's trailing zero bytes; anything else as it is.
fn unloaded_value(output: &ColumnType, quoted: &str) -> String {
match output {
ColumnType::TimestampTz | ColumnType::TimestampNtz => {
format!("fromUnixTimestamp64Micro(CAST({quoted} AS Int64), 'UTC')")
}
ColumnType::Binary => format!("substring({quoted}, 1)"),
_ => quoted.to_owned(),
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,47 +48,7 @@ impl ClickHouseWriter {
)));
}
self.settle_rows().await?;
let builder = s3_builder(
bucket,
region.as_deref(),
endpoint.as_deref(),
profile.as_deref(),
)
.map_err(|error| {
LoaderError::Invalid(format!(
"cannot sign ClickHouse's reads of s3://{bucket}/: {error}"
))
})?;
let setting = |key: AmazonS3ConfigKey| builder.get_config_value(&key);
let region = setting(AmazonS3ConfigKey::Region);
let endpoint =
setting(AmazonS3ConfigKey::S3Endpoint).or(setting(AmazonS3ConfigKey::Endpoint));
let in_endpoint = endpoint.is_some()
&& switched_on(setting(AmazonS3ConfigKey::VirtualHostedStyleRequest));
let unsigned = switched_on(setting(AmazonS3ConfigKey::SkipSignature));
let unreadable = |error: String| {
LoaderError::Invalid(format!(
"cannot sign ClickHouse's reads of s3://{bucket}/: {error}"
))
};
let credentials = match unsigned {
true => None,
false => {
let store = builder
.build()
.map_err(|error| unreadable(error.to_string()))?;
let credential = store
.credentials()
.get_credential()
.await
.map_err(|error| unreadable(error.to_string()))?;
Some(PullCredentials {
key_id: credential.key_id.clone(),
secret: credential.secret_key.clone(),
token: credential.token.clone(),
})
}
};
let (region, endpoint, credentials) = s3_reads(bucket, (region, endpoint, profile)).await?;
let parameters = credential_parameters(credentials.as_ref());
let target = &batch.target;
let (into, columns) = match (target.staging, target.builds) {
Expand All @@ -107,7 +67,7 @@ impl ClickHouseWriter {
let url = object_url(
(bucket, &key),
region.as_deref(),
(endpoint.as_deref(), in_endpoint),
(endpoint.0.as_deref(), endpoint.1),
)
.map_err(LoaderError::Invalid)?;
let file = pulled_file(source, data, url, (batch, columns))?;
Expand All @@ -131,6 +91,54 @@ impl ClickHouseWriter {
}
}

/// How ClickHouse reads objects of `bucket`, as SQLLoad's own S3 reads would (`region`, `endpoint` and `profile` from the connection): the region, the endpoint with whether the bucket is already in it (virtual-hosted requests to a custom endpoint), and the credentials SQLLoad resolves to sign `s3()` with, `None` for unsigned reads.
pub(in crate::adapters) async fn s3_reads(
bucket: &str,
(region, endpoint, profile): (&Option<String>, &Option<String>, &Option<String>),
) -> Result<
(
Option<String>,
(Option<String>, bool),
Option<PullCredentials>,
),
LoaderError,
> {
let unreadable = |error: String| {
LoaderError::Invalid(format!(
"cannot sign ClickHouse's reads of s3://{bucket}/: {error}"
))
};
let builder = s3_builder(
bucket,
region.as_deref(),
endpoint.as_deref(),
profile.as_deref(),
)
.map_err(|error| unreadable(error.to_string()))?;
let setting = |key: AmazonS3ConfigKey| builder.get_config_value(&key);
let region = setting(AmazonS3ConfigKey::Region);
let endpoint = setting(AmazonS3ConfigKey::S3Endpoint).or(setting(AmazonS3ConfigKey::Endpoint));
let in_endpoint =
endpoint.is_some() && switched_on(setting(AmazonS3ConfigKey::VirtualHostedStyleRequest));
if switched_on(setting(AmazonS3ConfigKey::SkipSignature)) {
return Ok((region, (endpoint, in_endpoint), None));
}
let store = builder
.build()
.map_err(|error| unreadable(error.to_string()))?;
let credential = store
.credentials()
.get_credential()
.await
.map_err(|error| unreadable(error.to_string()))?;
let credentials = PullCredentials {
key_id: credential.key_id.clone(),
secret: credential.secret_key.clone(),
token: credential.token.clone(),
};
Ok((region, (endpoint, in_endpoint), Some(credentials)))
}

/// Whether an S3 switch is on, read as object_store reads it from the environment (`true`, `1`, `yes`, `on` or `y`, in any case).
fn switched_on(value: Option<String>) -> bool {
value.is_some_and(|value| {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub(crate) mod arrow_batch;
pub(crate) mod direct_load;
pub(crate) mod table_writer;
pub(crate) mod unloaded_load;
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
//! A warehouse source's unloaded files loaded by ClickHouse reading them from S3 itself.

use crate::adapters::clickhouse::_helpers::dialect::pull::{credential_parameters, object_url};
use crate::adapters::clickhouse::_helpers::dialect::unloaded::{UnloadedFile, unloaded_insert};
use crate::adapters::clickhouse::_helpers::session::http::written_rows;
use crate::adapters::clickhouse::_helpers::write::direct_load::s3_reads;
use crate::adapters::clickhouse::models::{ClickHouseSql, ClickHouseWriter};
use crate::adapters::contract::models::{DirectLoaded, UnloadedBatch, UnloadedSource};
use crate::adapters::contract::types::{SqlAdapter, TableWriter};
use crate::column_types::models::ColumnType;
use crate::loader::errors::LoaderError;
use crate::store_paths::models::StoreKind;

impl ClickHouseWriter {
/// Load `batch` by having ClickHouse read the unloaded files from S3 itself, one `INSERT ... SELECT FROM s3()` per file, into the staging or build table the load publishes from, signed with the credentials SQLLoad reads S3 with. ClickHouse must load as many rows as the unload reports, or the batch fails before anything publishes.
pub(in crate::adapters) async fn pull_unloaded(
&mut self,
batch: &UnloadedBatch<'_>,
) -> Result<DirectLoaded, LoaderError> {
let UnloadedSource::Files(files) = batch.source else {
return Err(LoaderError::Invalid(
"ClickHouse reads a warehouse source's rows only from its unloaded files"
.to_owned(),
));
};
let StoreKind::S3 {
bucket,
region,
endpoint,
profile,
} = &files.folder.store
else {
return Err(LoaderError::Invalid(format!(
"ClickHouse reads unloaded files only from S3, not {}",
files.folder
)));
};
if files.keys.is_empty() {
return Ok(DirectLoaded::Staged(0));
}
self.settle_rows().await?;
let (region, endpoint, credentials) = s3_reads(bucket, (region, endpoint, profile)).await?;
let parameters = credential_parameters(credentials.as_ref());
let target = &batch.target;
let (into, columns) = match (target.staging, target.builds) {
(Some(staging), false) => (staging, target.columns),
(staging, _) => (staging.unwrap_or(target.table), batch.write_columns),
};
let builds = target.builds;
let sql_type = |column_type: &ColumnType| match builds {
true => ClickHouseSql.render_loader_logical_type(column_type).sql,
false => ClickHouseSql.render_staging_type(column_type),
};
let into = ClickHouseSql.render_qualified_name(into);
let mut rows = 0;
for (index, key) in files.keys.iter().enumerate() {
let object = format!("{}{key}", files.folder.prefix);
let url = object_url(
(bucket, &object),
region.as_deref(),
(endpoint.0.as_deref(), endpoint.1),
)
.map_err(LoaderError::Invalid)?;
let insert = unloaded_insert(
(&into, columns, &sql_type),
target,
(batch.read, batch.loaded_at),
(&UnloadedFile { url: &url, index }, credentials.as_ref()),
);
rows += written_rows(&self.destination, &insert, &parameters)
.await
.map_err(|error| LoaderError::Invalid(format!("{}{key}: {error}", files.folder)))?;
}
if rows != files.rows {
return Err(LoaderError::Invalid(format!(
"ClickHouse loaded {rows} rows from the unloaded files in {}, but the unload reports {}; the batch is not committed",
files.folder, files.rows
)));
}
Ok(DirectLoaded::Staged(rows))
}
}
2 changes: 2 additions & 0 deletions crates/sqlload/src/adapters/clickhouse/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,5 @@ mod pull;
mod rows;
#[path = "tests/test_types.rs"]
mod test_types;
#[path = "tests/test_unloaded.rs"]
mod unloaded;
8 changes: 8 additions & 0 deletions crates/sqlload/src/adapters/clickhouse/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,11 @@ pub(crate) struct SortedInsertTestCase {
pub(crate) key_columns: &'static [&'static str],
pub(crate) expected_insert: &'static str,
}

pub(crate) struct UnloadedInsertTestCase {
pub(crate) description: &'static str,
pub(crate) columns: Vec<crate::schema_plan::models::DesiredColumn>,
/// Whether the read is signed, or unsigned for a public bucket.
pub(crate) signed: bool,
pub(crate) expected_insert: &'static str,
}
Loading
Loading