Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ humantime = "2"
console = { version = "0.16", default-features = false }
indicatif = { version = "0.18", default-features = false, features = ["unicode-width"] }
nix = { version = "0.31", default-features = false, features = ["resource"] }
object_store = { version = "0.14", default-features = false, features = ["aws-base", "reqwest", "ring", "fs"] }
object_store = { version = "0.14", default-features = false, features = ["aws-base", "gcp-base", "reqwest", "ring", "fs"] }
libc = "0.2"
jsonwebtoken = { version = "11", default-features = false, features = ["use_pem", "rust_crypto"] }
parquet = { version = "60", default-features = false, features = ["arrow", "zstd", "async", "object_store", "snap", "lz4", "flate2", "flate2-rust_backend"] }
Expand Down
9 changes: 9 additions & 0 deletions crates/sqlload-testkit/src/fake_gcs/constants.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
/// fake-gcs-server as object_store tests against it, with the XML API's object requests; pinned by digest, as it has only a `latest` tag.
pub const IMAGE: &str = "tustvold/fake-gcs-server";
pub const TAG: &str =
"latest@sha256:dcd3aeacc07c731f1336e90c2889be2af8626ae993ee1fe2c0ba042ebbeb5a04";
pub const PORT: u16 = 4443;
/// The bucket every test uses, made when the server starts.
pub const BUCKET: &str = "example-bucket-test";
/// A service-account key that turns off OAuth, so the client sends no token the fake would have to check.
pub const SERVICE_ACCOUNT_KEY: &str = r#"{"private_key": "", "private_key_id": "", "client_email": "test@example.com", "disable_oauth": true}"#;
1 change: 1 addition & 0 deletions crates/sqlload-testkit/src/fake_gcs/main/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub mod start_fake_gcs;
37 changes: 37 additions & 0 deletions crates/sqlload-testkit/src/fake_gcs/main/start_fake_gcs.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
//! Start a fake GCS server.

use testcontainers::core::{IntoContainerPort, WaitFor};
use testcontainers::runners::AsyncRunner;
use testcontainers::{GenericImage, ImageExt};

use crate::fake_gcs::constants::{BUCKET, IMAGE, PORT, TAG};
use crate::fake_gcs::models::FakeGcs;

/// A fake GCS server over plain HTTP on a free host port, holding [`BUCKET`] (seeded with one `.keep` object outside every test's folder). It answers the XML API only for requests to its public host, `127.0.0.1`, which it matches whatever the port.
pub async fn start_fake_gcs() -> Result<FakeGcs, String> {
let container = GenericImage::new(IMAGE, TAG)
.with_exposed_port(PORT.tcp())
.with_wait_for(WaitFor::message_on_stderr("server started at"))
.with_copy_to(format!("/data/{BUCKET}/.keep"), b"keep".to_vec())
.with_cmd([
"-scheme",
"http",
"-backend",
"memory",
"-data",
"/data",
"-public-host",
"127.0.0.1",
])
.start()
.await
.map_err(|error| format!("cannot start the fake GCS server: {error}"))?;
let port = container
.get_host_port_ipv4(PORT.tcp())
.await
.map_err(|error| error.to_string())?;
Ok(FakeGcs {
endpoint: format!("http://127.0.0.1:{port}"),
container,
})
}
5 changes: 5 additions & 0 deletions crates/sqlload-testkit/src/fake_gcs/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
//! A fake GCS server serving the XML and JSON APIs SQLLoad's GCS client uses, standing in for Google Cloud Storage.

pub mod constants;
pub mod main;
pub mod models;
9 changes: 9 additions & 0 deletions crates/sqlload-testkit/src/fake_gcs/models.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
use testcontainers::{ContainerAsync, GenericImage};

/// A running fake GCS server; the container stops when this is dropped.
#[derive(Debug)]
pub struct FakeGcs {
/// Its base URL, `http://127.0.0.1:<port>`.
pub endpoint: String,
pub container: ContainerAsync<GenericImage>,
}
3 changes: 2 additions & 1 deletion crates/sqlload-testkit/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! Test doubles for sqlload: a fake Snowflake service, Redpanda, Postgres, MySQL and SQL Server containers, configurations and an exporter harness.
//! Test doubles for sqlload: a fake Snowflake service, a fake GCS server, Redpanda, Postgres, MySQL and SQL Server containers, configurations and an exporter harness.

#![forbid(unsafe_code)]

Expand All @@ -7,6 +7,7 @@ pub mod configs;
pub mod doc_pages;
pub mod fake_bigquery;
pub mod fake_databricks;
pub mod fake_gcs;
pub mod fake_sheets;
pub mod fake_snowflake;
pub mod harness;
Expand Down
7 changes: 5 additions & 2 deletions crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
//! Whether an S3 store points at a custom endpoint, which a warehouse reading S3 itself would not follow.
//! Whether an S3 or GCS store points at a custom endpoint, which a warehouse reading S3 itself would not follow.

use crate::store_paths::models::StoreKind;

/// Why an S3 store points somewhere a warehouse reading S3 itself would not read: an endpoint in the connection or in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows.
/// Why an S3 or GCS store points somewhere a warehouse reading it itself would not read: an endpoint in the connection (S3 or GCS) or, for S3, in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows.
pub(in crate::adapters) fn custom_endpoint(
store: &StoreKind,
variable: &dyn Fn(&str) -> Option<String>,
Expand All @@ -17,6 +17,9 @@ pub(in crate::adapters) fn custom_endpoint(
StoreKind::S3 { .. } if from_env => {
Some("AWS_ENDPOINT or AWS_ENDPOINT_URL sets a custom S3 endpoint".to_owned())
}
StoreKind::Gcs {
endpoint: Some(_), ..
} => Some("the GCS connection sets a custom endpoint".to_owned()),
_ => None,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,15 @@ use crate::adapters::databricks::constants::OUTSIDE_LOCATIONS;
use crate::adapters::databricks::models::DatabricksDestination;
use crate::routing::models::{FileFormat, Store, WriterAccepts};

/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads; and Parquet files in S3 under one of its external locations, which `COPY INTO` reads where they lie. Parquet files in S3 no external location covers are refused, never read by SQLLoad.
/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads; and Parquet files in S3 under one of its external locations, which `COPY INTO` reads where they lie (only S3 ones: SQLLoad checks that `COPY INTO` keeps every value exactly only for S3, so files under a GCS or Azure location go through the volume). Parquet files in S3 no external location covers are refused, never read by SQLLoad.
pub(in crate::adapters) fn writer_accepts(destination: &DatabricksDestination) -> WriterAccepts {
let volume = Store::InternalStage(destination.volume.clone());
let locations = destination
.external_locations
.get()
.into_iter()
.flatten()
.filter(|url| url.starts_with("s3://"))
.map(|url| Store::ObjectStore(url.clone()));
WriterAccepts {
arrow: false,
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlload/src/adapters/databricks/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,5 @@ mod stale_removals;
mod test_types;
#[path = "tests/test_unloaded_copy.rs"]
mod unloaded_copy;
#[path = "tests/test_writer_accepts.rs"]
mod writer_accepts;
7 changes: 7 additions & 0 deletions crates/sqlload/src/adapters/databricks/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,10 @@ pub(crate) struct UnloadedCopyTestCase {
pub(crate) builds: bool,
pub(crate) expected_statement: &'static str,
}

/// The external locations Databricks lists, and those it then reads files from itself, after its staging volume.
pub(crate) struct WriterAcceptsTestCase {
pub(crate) description: &'static str,
pub(crate) locations: &'static [&'static str],
pub(crate) expected_pulls_from: &'static [&'static str],
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
use std::sync::Arc;

use crate::adapters::databricks::main::stage::_writer_accepts::writer_accepts;
use crate::adapters::databricks::models::DatabricksDestination;
use crate::adapters::databricks::tests::test_types::WriterAcceptsTestCase;
use crate::auth::models::Secret;
use crate::databricks_api::models::DatabricksClient;
use crate::http::main::install_tls_provider::install_tls_provider;
use crate::routing::models::Store;

#[test]
fn given_external_locations_when_databricks_says_what_it_reads_itself_then_only_s3_ones_count() {
let test_cases = [
WriterAcceptsTestCase {
description: "an S3 location is read where it lies",
locations: &["s3://lake/raw/"],
expected_pulls_from: &["s3://lake/raw/"],
},
WriterAcceptsTestCase {
description: "a GCS location on Databricks on Google Cloud is left to SQLLoad, which checks what COPY INTO would change only for S3",
locations: &["gs://lake/raw/", "s3://lake/raw/"],
expected_pulls_from: &["s3://lake/raw/"],
},
];
install_tls_provider();
for test_case in test_cases {
let destination = DatabricksDestination {
api: DatabricksClient::new(
reqwest::Client::new(),
"example.cloud.databricks.com",
"w1",
("id", Secret(String::new())),
),
catalog: "main".to_owned(),
volume: "/Volumes/main/raw/stage".to_owned(),
ddl: Arc::default(),
external_locations: Arc::new(tokio::sync::OnceCell::new_with(Some(
test_case
.locations
.iter()
.map(|url| (*url).to_owned())
.collect(),
))),
};
assert_eq!(
writer_accepts(&destination).pulls_from,
std::iter::once(Store::InternalStage("/Volumes/main/raw/stage".to_owned()))
.chain(
test_case
.expected_pulls_from
.iter()
.map(|url| Store::ObjectStore((*url).to_owned()))
)
.collect::<Vec<Store>>(),
"{}",
test_case.description
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ pub fn connection_warehouse(connection: &ConnectionConfig) -> Option<(Warehouse,
ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. } => {
Some((Warehouse::DuckDb, None))
}
ConnectionConfig::S3 { .. } | ConnectionConfig::Filesystem { .. } => {
Some((Warehouse::Parquet, None))
}
ConnectionConfig::S3 { .. }
| ConnectionConfig::Gcs { .. }
| ConnectionConfig::Filesystem { .. } => Some((Warehouse::Parquet, None)),
ConnectionConfig::Postgres { database, .. } => {
Some((Warehouse::Postgres, Some(database.clone())))
}
Expand Down
31 changes: 24 additions & 7 deletions crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,18 @@ use crate::loader::models::FilesSource;
use crate::routing::models::Store;
use crate::store_paths::models::StoreKind;

/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix.
/// Check, at plan, that `destination` reads the files of `source` in S3 or GCS where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers, and DuckDB reads it from GCS (Databricks reads only S3). An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix.
pub async fn check_reach(
destination: &Destination,
(source, unload_to): (Option<&FilesSource>, &[Store]),
read: Option<&FilesRead>,
) -> Result<(), String> {
let source = source.filter(|source| matches!(source.location.store, StoreKind::S3 { .. }));
let source = source.filter(|source| {
matches!(
source.location.store,
StoreKind::S3 { .. } | StoreKind::Gcs { .. }
)
});
let unloads = unload_to
.iter()
.any(|store| matches!(store, Store::ObjectStore(url) if url.starts_with("s3://")));
Expand All @@ -30,7 +35,9 @@ pub async fn check_reach(
let first = read.and_then(|read| read.files.first());
let folder = source.location.to_string();
match destination {
Destination::Databricks(databricks) => {
Destination::Databricks(databricks)
if matches!(source.location.store, StoreKind::S3 { .. }) =>
{
databricks_reach(databricks, first.map(|_| folder.as_str())).await
}
#[cfg(feature = "duckdb")]
Expand All @@ -39,13 +46,23 @@ pub async fn check_reach(
return Ok(());
};
let url = format!("{folder}{}", data.file.relative_key);
let gcs = matches!(source.location.store, StoreKind::Gcs { .. });
check_files(duckdb, &source.location.store, &url)
.await
.map_err(|error| {
format!(
"{url} cannot be read by MotherDuck on its servers: {}. Fix: MotherDuck's servers read the bucket from their own network, with the keys SQLLoad's S3 reads use (the AWS_* variables, else the connection's profile or AWS_PROFILE, else the AWS credential chain on this machine): give those keys s3:GetObject and s3:ListBucket on it, and kms:Decrypt on its KMS key if it has one, and let the bucket policy allow reads from outside your network or VPC endpoint; a bucket only your network may reach can't load into MotherDuck",
first_line(&error.to_string())
)
let text = error.to_string();
let error = first_line(&text);
match (gcs, duckdb.motherduck) {
(true, true) => format!(
"{url} cannot be read by MotherDuck on its servers: {error}. Fix: MotherDuck's servers read the bucket from their own network with the gcs connection's HMAC key: check that hmac_key_id and hmac_secret are one active key, give its service account roles/storage.objectViewer on the bucket, and let no VPC Service Controls perimeter keep MotherDuck out"
),
(true, false) => format!(
"{url} cannot be read by DuckDB: {error}. Fix: DuckDB reads GCS with the gcs connection's HMAC key, not its service_account_key: check that hmac_key_id and hmac_secret are one active key, and give its service account roles/storage.objectViewer on the bucket"
),
(false, _) => format!(
"{url} cannot be read by MotherDuck on its servers: {error}. Fix: MotherDuck's servers read the bucket from their own network, with the keys SQLLoad's S3 reads use (the AWS_* variables, else the connection's profile or AWS_PROFILE, else the AWS credential chain on this machine): give those keys s3:GetObject and s3:ListBucket on it, and kms:Decrypt on its KMS key if it has one, and let the bucket policy allow reads from outside your network or VPC endpoint; a bucket only your network may reach can't load into MotherDuck"
),
}
})
}
_ => Ok(()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@ use crate::adapters::snowflake::models::StageRoute;
use crate::routing::models::WriterAccepts;
use crate::routing::models::{FileFormat, Store};

/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3 or on this machine itself, MotherDuck Parquet in S3 on its servers, and ClickHouse Parquet in S3), or what its stage route takes.
/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself, MotherDuck Parquet in S3 or GCS on its servers, and ClickHouse Parquet in S3), or what its stage route takes.
pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> WriterAccepts {
match destination {
#[cfg(feature = "duckdb")]
Destination::DuckDb(duckdb) if duckdb.motherduck => WriterAccepts {
arrow: true,
pulls_from: vec![Store::ObjectStore("s3://".to_owned())],
pulls_from: vec![
Store::ObjectStore("s3://".to_owned()),
Store::ObjectStore("gs://".to_owned()),
],
pull_formats: vec![FileFormat::Parquet],
..WriterAccepts::default()
},
Expand All @@ -22,6 +25,7 @@ pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) ->
arrow: true,
pulls_from: vec![
Store::ObjectStore("s3://".to_owned()),
Store::ObjectStore("gs://".to_owned()),
Store::Local(String::new()),
],
pull_formats: vec![FileFormat::Parquet],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use crate::adapters::snowflake::main::stage::_forced_stage::forced_stage;
use crate::adapters::snowflake::models::SnowflakeAccount;
use crate::loader::errors::LoaderError;
use crate::project::models::{ConnectionConfig, ProjectConfig};
use crate::store_paths::models::StoreKind;
use crate::store_paths::main::connection_store::connection_store;

/// The open destination of target `name`.
pub async fn connect_destination(
Expand Down Expand Up @@ -57,11 +57,13 @@ pub async fn connect_destination(
Some(connection @ ConnectionConfig::ClickHouse { .. }) => {
Destination::ClickHouse(Box::new(connect_clickhouse(connection)?))
}
Some(connection @ (ConnectionConfig::S3 { .. } | ConnectionConfig::Filesystem { .. })) => {
files(connection, target).ok_or_else(|| {
LoaderError::Invalid(format!("target {name:?}: its connection holds no files"))
})?
}
Some(
connection @ (ConnectionConfig::S3 { .. }
| ConnectionConfig::Gcs { .. }
| ConnectionConfig::Filesystem { .. }),
) => files(connection, target).ok_or_else(|| {
LoaderError::Invalid(format!("target {name:?}: its connection holds no files"))
})?,
Some(other) => {
return Err(LoaderError::Invalid(format!(
"target {name:?}: {} destinations are not supported yet",
Expand All @@ -78,26 +80,12 @@ pub async fn connect_destination(
Ok(destination)
}

/// The files destination of `target`, from its `s3` or `filesystem` connection; `None` for another kind.
/// The files destination of `target`, from its `s3`, `gcs` or `filesystem` connection (the only kinds it is called for).
fn files(
connection: &ConnectionConfig,
target: &crate::project::models::TargetConfig,
) -> Option<Destination> {
let store = match connection {
ConnectionConfig::S3 {
bucket,
region,
endpoint,
profile,
} => StoreKind::S3 {
bucket: bucket.clone(),
region: region.clone(),
endpoint: endpoint.clone(),
profile: profile.clone(),
},
ConnectionConfig::Filesystem { root } => StoreKind::Filesystem { root: root.clone() },
_ => return None,
};
let store = connection_store(connection)?;
Some(Destination::Files(Box::new(FilesDestination {
connection: target.connection.clone(),
store,
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlload/src/adapters/dispatch/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub enum Destination {
BigQuery(Box<BigQueryDestination>),
Postgres(Box<PostgresDestination>),
ClickHouse(Box<ClickHouseDestination>),
/// Parquet files in an s3 or filesystem connection.
/// Parquet files in an s3, gcs or filesystem connection.
Files(Box<FilesDestination>),
}

Expand Down
Loading
Loading