diff --git a/Cargo.toml b/Cargo.toml index f10bf6ab8..8a451c1d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ humantime = "2" console = { version = "0.16", default-features = false } indicatif = { version = "0.18", default-features = false, features = ["unicode-width"] } nix = { version = "0.31", default-features = false, features = ["resource"] } -object_store = { version = "0.14", default-features = false, features = ["aws-base", "reqwest", "ring", "fs"] } +object_store = { version = "0.14", default-features = false, features = ["aws-base", "gcp-base", "reqwest", "ring", "fs"] } libc = "0.2" jsonwebtoken = { version = "11", default-features = false, features = ["use_pem", "rust_crypto"] } parquet = { version = "60", default-features = false, features = ["arrow", "zstd", "async", "object_store", "snap", "lz4", "flate2", "flate2-rust_backend"] } diff --git a/crates/sqlload-testkit/src/fake_gcs/constants.rs b/crates/sqlload-testkit/src/fake_gcs/constants.rs new file mode 100644 index 000000000..c2384aebe --- /dev/null +++ b/crates/sqlload-testkit/src/fake_gcs/constants.rs @@ -0,0 +1,9 @@ +/// fake-gcs-server as object_store tests against it, with the XML API's object requests; pinned by digest, as it has only a `latest` tag. +pub const IMAGE: &str = "tustvold/fake-gcs-server"; +pub const TAG: &str = + "latest@sha256:dcd3aeacc07c731f1336e90c2889be2af8626ae993ee1fe2c0ba042ebbeb5a04"; +pub const PORT: u16 = 4443; +/// The bucket every test uses, made when the server starts. +pub const BUCKET: &str = "example-bucket-test"; +/// A service-account key that turns off OAuth, so the client sends no token the fake would have to check. +pub const SERVICE_ACCOUNT_KEY: &str = r#"{"private_key": "", "private_key_id": "", "client_email": "test@example.com", "disable_oauth": true}"#; diff --git a/crates/sqlload-testkit/src/fake_gcs/main/mod.rs b/crates/sqlload-testkit/src/fake_gcs/main/mod.rs new file mode 100644 index 000000000..00cf272ef --- /dev/null +++ b/crates/sqlload-testkit/src/fake_gcs/main/mod.rs @@ -0,0 +1 @@ +pub mod start_fake_gcs; diff --git a/crates/sqlload-testkit/src/fake_gcs/main/start_fake_gcs.rs b/crates/sqlload-testkit/src/fake_gcs/main/start_fake_gcs.rs new file mode 100644 index 000000000..3a1bfa7b2 --- /dev/null +++ b/crates/sqlload-testkit/src/fake_gcs/main/start_fake_gcs.rs @@ -0,0 +1,37 @@ +//! Start a fake GCS server. + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; + +use crate::fake_gcs::constants::{BUCKET, IMAGE, PORT, TAG}; +use crate::fake_gcs::models::FakeGcs; + +/// A fake GCS server over plain HTTP on a free host port, holding [`BUCKET`] (seeded with one `.keep` object outside every test's folder). It answers the XML API only for requests to its public host, `127.0.0.1`, which it matches whatever the port. +pub async fn start_fake_gcs() -> Result { + let container = GenericImage::new(IMAGE, TAG) + .with_exposed_port(PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr("server started at")) + .with_copy_to(format!("/data/{BUCKET}/.keep"), b"keep".to_vec()) + .with_cmd([ + "-scheme", + "http", + "-backend", + "memory", + "-data", + "/data", + "-public-host", + "127.0.0.1", + ]) + .start() + .await + .map_err(|error| format!("cannot start the fake GCS server: {error}"))?; + let port = container + .get_host_port_ipv4(PORT.tcp()) + .await + .map_err(|error| error.to_string())?; + Ok(FakeGcs { + endpoint: format!("http://127.0.0.1:{port}"), + container, + }) +} diff --git a/crates/sqlload-testkit/src/fake_gcs/mod.rs b/crates/sqlload-testkit/src/fake_gcs/mod.rs new file mode 100644 index 000000000..c1fb706b4 --- /dev/null +++ b/crates/sqlload-testkit/src/fake_gcs/mod.rs @@ -0,0 +1,5 @@ +//! A fake GCS server serving the XML and JSON APIs SQLLoad's GCS client uses, standing in for Google Cloud Storage. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlload-testkit/src/fake_gcs/models.rs b/crates/sqlload-testkit/src/fake_gcs/models.rs new file mode 100644 index 000000000..e2dc69ec8 --- /dev/null +++ b/crates/sqlload-testkit/src/fake_gcs/models.rs @@ -0,0 +1,9 @@ +use testcontainers::{ContainerAsync, GenericImage}; + +/// A running fake GCS server; the container stops when this is dropped. +#[derive(Debug)] +pub struct FakeGcs { + /// Its base URL, `http://127.0.0.1:`. + pub endpoint: String, + pub container: ContainerAsync, +} diff --git a/crates/sqlload-testkit/src/lib.rs b/crates/sqlload-testkit/src/lib.rs index 189f815eb..ba0304746 100644 --- a/crates/sqlload-testkit/src/lib.rs +++ b/crates/sqlload-testkit/src/lib.rs @@ -1,4 +1,4 @@ -//! Test doubles for sqlload: a fake Snowflake service, Redpanda, Postgres, MySQL and SQL Server containers, configurations and an exporter harness. +//! Test doubles for sqlload: a fake Snowflake service, a fake GCS server, Redpanda, Postgres, MySQL and SQL Server containers, configurations and an exporter harness. #![forbid(unsafe_code)] @@ -7,6 +7,7 @@ pub mod configs; pub mod doc_pages; pub mod fake_bigquery; pub mod fake_databricks; +pub mod fake_gcs; pub mod fake_sheets; pub mod fake_snowflake; pub mod harness; diff --git a/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs b/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs index 667f3b6b3..3f758a0e6 100644 --- a/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs +++ b/crates/sqlload/src/adapters/contract/main/_custom_endpoint.rs @@ -1,8 +1,8 @@ -//! Whether an S3 store points at a custom endpoint, which a warehouse reading S3 itself would not follow. +//! Whether an S3 or GCS store points at a custom endpoint, which a warehouse reading S3 itself would not follow. use crate::store_paths::models::StoreKind; -/// Why an S3 store points somewhere a warehouse reading S3 itself would not read: an endpoint in the connection or in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows. +/// Why an S3 or GCS store points somewhere a warehouse reading it itself would not read: an endpoint in the connection (S3 or GCS) or, for S3, in `AWS_ENDPOINT` / `AWS_ENDPOINT_URL` (each as `variable` reads it from the environment, unset when empty, as SQLLoad's reader reads them), which SQLLoad's reader follows. pub(in crate::adapters) fn custom_endpoint( store: &StoreKind, variable: &dyn Fn(&str) -> Option, @@ -17,6 +17,9 @@ pub(in crate::adapters) fn custom_endpoint( StoreKind::S3 { .. } if from_env => { Some("AWS_ENDPOINT or AWS_ENDPOINT_URL sets a custom S3 endpoint".to_owned()) } + StoreKind::Gcs { + endpoint: Some(_), .. + } => Some("the GCS connection sets a custom endpoint".to_owned()), _ => None, } } diff --git a/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs b/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs index 43c8e0317..eca4fc11c 100644 --- a/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs +++ b/crates/sqlload/src/adapters/databricks/main/stage/_writer_accepts.rs @@ -4,7 +4,7 @@ use crate::adapters::databricks::constants::OUTSIDE_LOCATIONS; use crate::adapters::databricks::models::DatabricksDestination; use crate::routing::models::{FileFormat, Store, WriterAccepts}; -/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads; and Parquet files in S3 under one of its external locations, which `COPY INTO` reads where they lie. Parquet files in S3 no external location covers are refused, never read by SQLLoad. +/// Parquet files in the staging volume, which SQLLoad writes and `COPY INTO` reads; and Parquet files in S3 under one of its external locations, which `COPY INTO` reads where they lie (only S3 ones: SQLLoad checks that `COPY INTO` keeps every value exactly only for S3, so files under a GCS or Azure location go through the volume). Parquet files in S3 no external location covers are refused, never read by SQLLoad. pub(in crate::adapters) fn writer_accepts(destination: &DatabricksDestination) -> WriterAccepts { let volume = Store::InternalStage(destination.volume.clone()); let locations = destination @@ -12,6 +12,7 @@ pub(in crate::adapters) fn writer_accepts(destination: &DatabricksDestination) - .get() .into_iter() .flatten() + .filter(|url| url.starts_with("s3://")) .map(|url| Store::ObjectStore(url.clone())); WriterAccepts { arrow: false, diff --git a/crates/sqlload/src/adapters/databricks/tests.rs b/crates/sqlload/src/adapters/databricks/tests.rs index f0e6de2fd..5200ae477 100644 --- a/crates/sqlload/src/adapters/databricks/tests.rs +++ b/crates/sqlload/src/adapters/databricks/tests.rs @@ -10,3 +10,5 @@ mod stale_removals; mod test_types; #[path = "tests/test_unloaded_copy.rs"] mod unloaded_copy; +#[path = "tests/test_writer_accepts.rs"] +mod writer_accepts; diff --git a/crates/sqlload/src/adapters/databricks/tests/test_types.rs b/crates/sqlload/src/adapters/databricks/tests/test_types.rs index 1f7d1cb4d..b780dd395 100644 --- a/crates/sqlload/src/adapters/databricks/tests/test_types.rs +++ b/crates/sqlload/src/adapters/databricks/tests/test_types.rs @@ -41,3 +41,10 @@ pub(crate) struct UnloadedCopyTestCase { pub(crate) builds: bool, pub(crate) expected_statement: &'static str, } + +/// The external locations Databricks lists, and those it then reads files from itself, after its staging volume. +pub(crate) struct WriterAcceptsTestCase { + pub(crate) description: &'static str, + pub(crate) locations: &'static [&'static str], + pub(crate) expected_pulls_from: &'static [&'static str], +} diff --git a/crates/sqlload/src/adapters/databricks/tests/test_writer_accepts.rs b/crates/sqlload/src/adapters/databricks/tests/test_writer_accepts.rs new file mode 100644 index 000000000..d217c03fc --- /dev/null +++ b/crates/sqlload/src/adapters/databricks/tests/test_writer_accepts.rs @@ -0,0 +1,59 @@ +use std::sync::Arc; + +use crate::adapters::databricks::main::stage::_writer_accepts::writer_accepts; +use crate::adapters::databricks::models::DatabricksDestination; +use crate::adapters::databricks::tests::test_types::WriterAcceptsTestCase; +use crate::auth::models::Secret; +use crate::databricks_api::models::DatabricksClient; +use crate::http::main::install_tls_provider::install_tls_provider; +use crate::routing::models::Store; + +#[test] +fn given_external_locations_when_databricks_says_what_it_reads_itself_then_only_s3_ones_count() { + let test_cases = [ + WriterAcceptsTestCase { + description: "an S3 location is read where it lies", + locations: &["s3://lake/raw/"], + expected_pulls_from: &["s3://lake/raw/"], + }, + WriterAcceptsTestCase { + description: "a GCS location on Databricks on Google Cloud is left to SQLLoad, which checks what COPY INTO would change only for S3", + locations: &["gs://lake/raw/", "s3://lake/raw/"], + expected_pulls_from: &["s3://lake/raw/"], + }, + ]; + install_tls_provider(); + for test_case in test_cases { + let destination = DatabricksDestination { + api: DatabricksClient::new( + reqwest::Client::new(), + "example.cloud.databricks.com", + "w1", + ("id", Secret(String::new())), + ), + catalog: "main".to_owned(), + volume: "/Volumes/main/raw/stage".to_owned(), + ddl: Arc::default(), + external_locations: Arc::new(tokio::sync::OnceCell::new_with(Some( + test_case + .locations + .iter() + .map(|url| (*url).to_owned()) + .collect(), + ))), + }; + assert_eq!( + writer_accepts(&destination).pulls_from, + std::iter::once(Store::InternalStage("/Volumes/main/raw/stage".to_owned())) + .chain( + test_case + .expected_pulls_from + .iter() + .map(|url| Store::ObjectStore((*url).to_owned())) + ) + .collect::>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/dispatch/main/dialect/connection_warehouse.rs b/crates/sqlload/src/adapters/dispatch/main/dialect/connection_warehouse.rs index cea45a400..3423d8679 100644 --- a/crates/sqlload/src/adapters/dispatch/main/dialect/connection_warehouse.rs +++ b/crates/sqlload/src/adapters/dispatch/main/dialect/connection_warehouse.rs @@ -17,9 +17,9 @@ pub fn connection_warehouse(connection: &ConnectionConfig) -> Option<(Warehouse, ConnectionConfig::DuckDb { .. } | ConnectionConfig::MotherDuck { .. } => { Some((Warehouse::DuckDb, None)) } - ConnectionConfig::S3 { .. } | ConnectionConfig::Filesystem { .. } => { - Some((Warehouse::Parquet, None)) - } + ConnectionConfig::S3 { .. } + | ConnectionConfig::Gcs { .. } + | ConnectionConfig::Filesystem { .. } => Some((Warehouse::Parquet, None)), ConnectionConfig::Postgres { database, .. } => { Some((Warehouse::Postgres, Some(database.clone()))) } diff --git a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs index 3309950cc..8bf88c5c7 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/check_reach.rs @@ -11,13 +11,18 @@ use crate::loader::models::FilesSource; use crate::routing::models::Store; use crate::store_paths::models::StoreKind; -/// Check, at plan, that `destination` reads the files of `source` in S3 where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers. An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix. +/// Check, at plan, that `destination` reads the files of `source` in S3 or GCS where they lie, or the S3 folder a warehouse source unloads to (`unload_to`), for a table nothing sends through SQLLoad, with `read` its planned read: Databricks looks up its Unity Catalog external locations once per run, which [`writer_accepts`](crate::adapters::dispatch::main::route::writer_accepts::writer_accepts) then offers, and lists the folder when one covers it and it holds files to load; MotherDuck reads the first file to load on its servers, and DuckDB reads it from GCS (Databricks reads only S3). An unload's folder is new each run, so for it Databricks only lists its locations, and the route then needs one to cover it. The error says why it can't, with the fix. pub async fn check_reach( destination: &Destination, (source, unload_to): (Option<&FilesSource>, &[Store]), read: Option<&FilesRead>, ) -> Result<(), String> { - let source = source.filter(|source| matches!(source.location.store, StoreKind::S3 { .. })); + let source = source.filter(|source| { + matches!( + source.location.store, + StoreKind::S3 { .. } | StoreKind::Gcs { .. } + ) + }); let unloads = unload_to .iter() .any(|store| matches!(store, Store::ObjectStore(url) if url.starts_with("s3://"))); @@ -30,7 +35,9 @@ pub async fn check_reach( let first = read.and_then(|read| read.files.first()); let folder = source.location.to_string(); match destination { - Destination::Databricks(databricks) => { + Destination::Databricks(databricks) + if matches!(source.location.store, StoreKind::S3 { .. }) => + { databricks_reach(databricks, first.map(|_| folder.as_str())).await } #[cfg(feature = "duckdb")] @@ -39,13 +46,23 @@ pub async fn check_reach( return Ok(()); }; let url = format!("{folder}{}", data.file.relative_key); + let gcs = matches!(source.location.store, StoreKind::Gcs { .. }); check_files(duckdb, &source.location.store, &url) .await .map_err(|error| { - format!( - "{url} cannot be read by MotherDuck on its servers: {}. Fix: MotherDuck's servers read the bucket from their own network, with the keys SQLLoad's S3 reads use (the AWS_* variables, else the connection's profile or AWS_PROFILE, else the AWS credential chain on this machine): give those keys s3:GetObject and s3:ListBucket on it, and kms:Decrypt on its KMS key if it has one, and let the bucket policy allow reads from outside your network or VPC endpoint; a bucket only your network may reach can't load into MotherDuck", - first_line(&error.to_string()) - ) + let text = error.to_string(); + let error = first_line(&text); + match (gcs, duckdb.motherduck) { + (true, true) => format!( + "{url} cannot be read by MotherDuck on its servers: {error}. Fix: MotherDuck's servers read the bucket from their own network with the gcs connection's HMAC key: check that hmac_key_id and hmac_secret are one active key, give its service account roles/storage.objectViewer on the bucket, and let no VPC Service Controls perimeter keep MotherDuck out" + ), + (true, false) => format!( + "{url} cannot be read by DuckDB: {error}. Fix: DuckDB reads GCS with the gcs connection's HMAC key, not its service_account_key: check that hmac_key_id and hmac_secret are one active key, and give its service account roles/storage.objectViewer on the bucket" + ), + (false, _) => format!( + "{url} cannot be read by MotherDuck on its servers: {error}. Fix: MotherDuck's servers read the bucket from their own network, with the keys SQLLoad's S3 reads use (the AWS_* variables, else the connection's profile or AWS_PROFILE, else the AWS credential chain on this machine): give those keys s3:GetObject and s3:ListBucket on it, and kms:Decrypt on its KMS key if it has one, and let the bucket policy allow reads from outside your network or VPC endpoint; a bucket only your network may reach can't load into MotherDuck" + ), + } }) } _ => Ok(()), diff --git a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs index 09e0c9289..3f42d42f0 100644 --- a/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs +++ b/crates/sqlload/src/adapters/dispatch/main/route/writer_accepts.rs @@ -7,13 +7,16 @@ use crate::adapters::snowflake::models::StageRoute; use crate::routing::models::WriterAccepts; use crate::routing::models::{FileFormat, Store}; -/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3 or on this machine itself, MotherDuck Parquet in S3 on its servers, and ClickHouse Parquet in S3), or what its stage route takes. +/// What `destination` takes over the chosen `stage`: Arrow batches (DuckDB also reads Parquet in S3, GCS or on this machine itself, MotherDuck Parquet in S3 or GCS on its servers, and ClickHouse Parquet in S3), or what its stage route takes. pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> WriterAccepts { match destination { #[cfg(feature = "duckdb")] Destination::DuckDb(duckdb) if duckdb.motherduck => WriterAccepts { arrow: true, - pulls_from: vec![Store::ObjectStore("s3://".to_owned())], + pulls_from: vec![ + Store::ObjectStore("s3://".to_owned()), + Store::ObjectStore("gs://".to_owned()), + ], pull_formats: vec![FileFormat::Parquet], ..WriterAccepts::default() }, @@ -22,6 +25,7 @@ pub fn writer_accepts(destination: &Destination, stage: Option<&StageRoute>) -> arrow: true, pulls_from: vec![ Store::ObjectStore("s3://".to_owned()), + Store::ObjectStore("gs://".to_owned()), Store::Local(String::new()), ], pull_formats: vec![FileFormat::Parquet], diff --git a/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs b/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs index 876df3b46..58d2afa1f 100644 --- a/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs +++ b/crates/sqlload/src/adapters/dispatch/main/session/connect_destination.rs @@ -14,7 +14,7 @@ use crate::adapters::snowflake::main::stage::_forced_stage::forced_stage; use crate::adapters::snowflake::models::SnowflakeAccount; use crate::loader::errors::LoaderError; use crate::project::models::{ConnectionConfig, ProjectConfig}; -use crate::store_paths::models::StoreKind; +use crate::store_paths::main::connection_store::connection_store; /// The open destination of target `name`. pub async fn connect_destination( @@ -57,11 +57,13 @@ pub async fn connect_destination( Some(connection @ ConnectionConfig::ClickHouse { .. }) => { Destination::ClickHouse(Box::new(connect_clickhouse(connection)?)) } - Some(connection @ (ConnectionConfig::S3 { .. } | ConnectionConfig::Filesystem { .. })) => { - files(connection, target).ok_or_else(|| { - LoaderError::Invalid(format!("target {name:?}: its connection holds no files")) - })? - } + Some( + connection @ (ConnectionConfig::S3 { .. } + | ConnectionConfig::Gcs { .. } + | ConnectionConfig::Filesystem { .. }), + ) => files(connection, target).ok_or_else(|| { + LoaderError::Invalid(format!("target {name:?}: its connection holds no files")) + })?, Some(other) => { return Err(LoaderError::Invalid(format!( "target {name:?}: {} destinations are not supported yet", @@ -78,26 +80,12 @@ pub async fn connect_destination( Ok(destination) } -/// The files destination of `target`, from its `s3` or `filesystem` connection; `None` for another kind. +/// The files destination of `target`, from its `s3`, `gcs` or `filesystem` connection (the only kinds it is called for). fn files( connection: &ConnectionConfig, target: &crate::project::models::TargetConfig, ) -> Option { - let store = match connection { - ConnectionConfig::S3 { - bucket, - region, - endpoint, - profile, - } => StoreKind::S3 { - bucket: bucket.clone(), - region: region.clone(), - endpoint: endpoint.clone(), - profile: profile.clone(), - }, - ConnectionConfig::Filesystem { root } => StoreKind::Filesystem { root: root.clone() }, - _ => return None, - }; + let store = connection_store(connection)?; Some(Destination::Files(Box::new(FilesDestination { connection: target.connection.clone(), store, diff --git a/crates/sqlload/src/adapters/dispatch/models.rs b/crates/sqlload/src/adapters/dispatch/models.rs index 11656f25b..56c308dd1 100644 --- a/crates/sqlload/src/adapters/dispatch/models.rs +++ b/crates/sqlload/src/adapters/dispatch/models.rs @@ -17,7 +17,7 @@ pub enum Destination { BigQuery(Box), Postgres(Box), ClickHouse(Box), - /// Parquet files in an s3 or filesystem connection. + /// Parquet files in an s3, gcs or filesystem connection. Files(Box), } diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs b/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs index 851ecd8dd..bae690583 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/direct_load.rs @@ -86,7 +86,7 @@ impl DuckDbWriter { fn file_url(location: &Location, key: &str) -> String { let path = format!("{location}{key}"); match location.store { - StoreKind::S3 { .. } | StoreKind::Sftp { .. } => path, + StoreKind::S3 { .. } | StoreKind::Gcs { .. } | StoreKind::Sftp { .. } => path, StoreKind::Filesystem { .. } => { std::path::absolute(&path).map_or(path, |absolute| absolute.display().to_string()) } diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs index 164bc959d..0d9dfd677 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs @@ -20,7 +20,7 @@ use crate::postgres_source::models::ReadColumn; use crate::schema_plan::constants::MERGE_ROW_COLUMN; use crate::schema_plan::models::DesiredColumn; use crate::stage_api::models::AwsCredentials; -use crate::store_paths::models::StoreKind; +use crate::store_paths::models::{HmacKey, StoreKind}; /// One file DuckDB reads: where it reads it, and the values SQLLoad gives every row of it (its name and version as the files source records them, and its partition values), by column (`None` for a partition folder the file lacks). #[derive(Debug, Clone, PartialEq, Eq)] @@ -264,7 +264,7 @@ pub(crate) fn default_region(variable: &dyn Fn(&str) -> Option) -> Strin .unwrap_or_else(|| "us-east-1".to_owned()) } -/// What DuckDB needs to read files in an S3 bucket itself: the extensions to load (installing them on first use) and the [temporary secret](s3_secret), with its parameters, whose keys come from the AWS credential chain when `chain`. +/// What DuckDB needs to read files in an S3 or GCS bucket itself: the extensions to load (installing them on first use) and the [temporary secret](s3_secret), with its parameters, whose keys come from the AWS credential chain when `chain`. pub(crate) struct BucketAccess { pub bucket: String, pub extensions: &'static str, @@ -294,16 +294,23 @@ pub(crate) fn secret_current(set: Option<&SetSecret>, wanted: &SetSecret, now: I }) } -/// For files in S3, [`BucketAccess`] for their bucket: the secret in the connection's region, else `AWS_REGION` / `AWS_DEFAULT_REGION` (unset when empty), else us-east-1 as SQLLoad's reader does; with the keys SQLLoad's own S3 reads use (the `AWS_*` variables, else the shared credentials file under the connection's `profile` or `AWS_PROFILE`), passed as parameters and never in SQL, otherwise the AWS credential chain, resolved where DuckDB runs. `None` for files elsewhere. +/// For files in S3, [`BucketAccess`] for their bucket: the secret in the connection's region, else `AWS_REGION` / `AWS_DEFAULT_REGION` (unset when empty), else us-east-1 as SQLLoad's reader does; with the keys SQLLoad's own S3 reads use (the `AWS_*` variables, else the shared credentials file under the connection's `profile` or `AWS_PROFILE`), passed as parameters and never in SQL, otherwise the AWS credential chain, resolved where DuckDB runs. For files in GCS, [`gcs_access`] when the connection has an HMAC key. `None` for files elsewhere. pub(crate) fn bucket_access(store: &StoreKind) -> Result, DuckDbError> { - let StoreKind::S3 { - bucket, - region, - profile, - .. - } = store - else { - return Ok(None); + let (bucket, region, profile) = match store { + StoreKind::S3 { + bucket, + region, + profile, + .. + } => (bucket, region, profile), + StoreKind::Gcs { + bucket, + hmac: Some(key), + .. + } => return Ok(Some(gcs_access(bucket, key))), + StoreKind::Gcs { .. } | StoreKind::Filesystem { .. } | StoreKind::Sftp { .. } => { + return Ok(None); + } }; let keys = s3_keys(profile.as_deref()) .map_err(DuckDbError::Task)? @@ -325,6 +332,24 @@ pub(crate) fn bucket_access(store: &StoreKind) -> Result, D })) } +/// For files in a GCS bucket, [`BucketAccess`] with the connection's HMAC key `key`, the only credential DuckDB reads GCS with: a temporary `gcs` secret scoped to the bucket and named after it (hex, as for S3), its key passed as parameters and never in SQL, kept apart from an S3 bucket of the same name. +fn gcs_access(bucket: &str, key: &HmacKey) -> BucketAccess { + let name: String = bucket.bytes().map(|byte| format!("{byte:02x}")).collect(); + BucketAccess { + bucket: format!("gs://{bucket}"), + extensions: "INSTALL httpfs; LOAD httpfs;", + statement: format!( + "CREATE OR REPLACE TEMPORARY SECRET sqlload_gcs_{name} (TYPE gcs, KEY_ID ?, SECRET ?, SCOPE ?)" + ), + parameters: vec![ + key.key_id.clone(), + key.secret.expose().to_owned(), + format!("gs://{bucket}/"), + ], + chain: false, + } +} + /// `keys` as the `KEY_ID`, `SECRET` and `SESSION_TOKEN` parameters of a DuckDB secret. fn secret_parameters(keys: AwsCredentials) -> [String; 3] { let token = keys.session_token.map(|token| token.0).unwrap_or_default(); diff --git a/crates/sqlload/src/adapters/duckdb/main/_check_files.rs b/crates/sqlload/src/adapters/duckdb/main/_check_files.rs index 6d3d29a28..807ba1faf 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_check_files.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_check_files.rs @@ -1,4 +1,4 @@ -//! Check that MotherDuck can read a files source's files on its servers. +//! Check that MotherDuck can read a files source's files on its servers, and DuckDB a GCS bucket's. use crate::adapters::contract::types::SqlAdapter; use crate::adapters::duckdb::errors::DuckDbError; @@ -7,17 +7,22 @@ use crate::adapters::duckdb::main::_pull_files::pull_files; use crate::adapters::duckdb::models::{DuckDbDestination, DuckDbSql}; use crate::store_paths::models::StoreKind; -/// Check that MotherDuck reads `url`, a Parquet file in `store`, on its servers, with the session's secret for the bucket as a load sets it; nothing for SQLLoad's own DuckDB, which reads with the same keys as SQLLoad's reader. +/// Check that MotherDuck reads `url`, a Parquet file in `store`, on its servers, or DuckDB reads it from GCS, with the session's secret for the bucket as a load sets it; nothing for SQLLoad's own DuckDB reading S3, with the same keys as SQLLoad's reader (in GCS it reads with the connection's HMAC key, which SQLLoad's reader doesn't use). pub(in crate::adapters) async fn check_files( destination: &DuckDbDestination, store: &StoreKind, url: &str, ) -> Result<(), DuckDbError> { - if !destination.motherduck { + let gcs = matches!(store, StoreKind::Gcs { .. }); + if !destination.motherduck && !gcs { return Ok(()); } + let remote = match destination.motherduck { + true => ", MD_RUN = REMOTE", + false => "", + }; let probe = format!( - "SELECT count(*) FROM read_parquet({}, hive_partitioning = false, MD_RUN = REMOTE)", + "SELECT count(*) FROM read_parquet({}, hive_partitioning = false{remote})", DuckDbSql.render_loader_value_literal(url) ); open_bucket(destination, store).await?; diff --git a/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs b/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs index 1aaa2388e..1f86c3f66 100644 --- a/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs +++ b/crates/sqlload/src/adapters/duckdb/main/_direct_problem.rs @@ -6,14 +6,19 @@ use crate::adapters::duckdb::_helpers::direct_checks::reserved_name; use crate::adapters::duckdb::_helpers::motherduck_connection::environment_variable; use crate::files_source::models::{FilesRead, SourceFormat}; use crate::loader::models::TableSpec; +use crate::store_paths::models::StoreKind; -/// Why DuckDB cannot load the planned `read` of a files table exactly by reading its Parquet files itself, so SQLLoad reads them instead; `None` when it can. DuckDB's casts differ from SQLLoad's (it rounds where SQLLoad truncates, and promotes files that disagree), so every column must load as the type the files hold it as; masks, custom S3 endpoints and names DuckDB would read as patterns also need SQLLoad's reader. +/// Why DuckDB cannot load the planned `read` of a files table exactly by reading its Parquet files itself, so SQLLoad reads them instead; `None` when it can. DuckDB's casts differ from SQLLoad's (it rounds where SQLLoad truncates, and promotes files that disagree), so every column must load as the type the files hold it as; masks, custom S3 and GCS endpoints, GCS without an HMAC key (DuckDB takes no service-account key) and names DuckDB would read as patterns also need SQLLoad's reader. pub(in crate::adapters) fn direct_problem(spec: &TableSpec, read: &FilesRead) -> Option { let files = spec.files.as_ref()?; let problems = [ (files.encoding.format != SourceFormat::Parquet) .then(|| "only Parquet files are read by DuckDB itself".to_owned()), custom_endpoint(&files.location.store, &environment_variable), + matches!(files.location.store, StoreKind::Gcs { hmac: None, .. }).then(|| { + "the gcs connection sets no hmac_key_id, which DuckDB and MotherDuck read GCS with" + .to_owned() + }), files.table.columns.iter().find_map(reserved_name), (!spec.masked.columns.is_empty()).then(|| "the table masks columns".to_owned()), files_direct_problem(files, read), diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs b/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs index 68b201aa6..3111e3d85 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_motherduck.rs @@ -349,21 +349,25 @@ fn given_a_duckdb_connection_when_its_database_is_a_motherduck_path_then_it_is_r } #[test] -fn given_a_motherduck_destination_when_routing_then_it_reads_parquet_only_from_s3_itself() +fn given_a_motherduck_destination_when_routing_then_it_reads_parquet_only_from_object_stores_itself() -> Result<(), String> { let test_cases = [ MotherDuckRouteTestCase { - description: "local DuckDB reads Parquet in S3 and on this machine itself", + description: "local DuckDB reads Parquet in S3, GCS and on this machine itself", motherduck: false, expected_pulls_from: vec![ Store::ObjectStore("s3://".to_owned()), + Store::ObjectStore("gs://".to_owned()), Store::Local(String::new()), ], }, MotherDuckRouteTestCase { - description: "MotherDuck reads Parquet in S3 on its servers, and never files on this machine", + description: "MotherDuck reads Parquet in S3 and GCS on its servers, and never files on this machine", motherduck: true, - expected_pulls_from: vec![Store::ObjectStore("s3://".to_owned())], + expected_pulls_from: vec![ + Store::ObjectStore("s3://".to_owned()), + Store::ObjectStore("gs://".to_owned()), + ], }, ]; for test_case in test_cases { diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs index fa57c39c6..1229b7d06 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_s3_secret.rs @@ -2,12 +2,13 @@ use crate::adapters::contract::main::_custom_endpoint::custom_endpoint; use std::time::{Duration, Instant}; use crate::adapters::duckdb::_helpers::pull_sql::{ - BucketAccess, default_region, s3_secret, secret_current, + BucketAccess, bucket_access, default_region, s3_secret, secret_current, }; use crate::adapters::duckdb::tests::test_types::{ - AwsVariablesTestCase, S3SecretTestCase, SecretCurrentTestCase, + AwsVariablesTestCase, GcsAccessTestCase, S3SecretTestCase, SecretCurrentTestCase, }; -use crate::store_paths::models::StoreKind; +use crate::auth::models::Secret; +use crate::store_paths::models::{HmacKey, StoreKind}; #[test] fn given_an_s3_bucket_when_duckdb_reads_its_files_itself_then_its_secret_carries_sqlloads_keys_only_as_parameters() @@ -176,3 +177,65 @@ fn given_a_buckets_secret_already_set_when_a_table_opens_the_bucket_then_it_is_s ); } } + +#[test] +fn given_a_gcs_bucket_when_duckdb_reads_its_files_itself_then_its_secret_carries_the_connections_hmac_key_only_as_parameters() +-> Result<(), String> { + let test_cases = [ + GcsAccessTestCase { + description: "an HMAC key becomes a GCS secret scoped to the bucket, named apart from an S3 bucket of the same name", + hmac: Some(("GOOG1EXAMPLE", "hm4c")), + expected_access: Some(( + "gs://lake", + "CREATE OR REPLACE TEMPORARY SECRET sqlload_gcs_6c616b65 (TYPE gcs, KEY_ID ?, SECRET ?, SCOPE ?)", + &["GOOG1EXAMPLE", "hm4c", "gs://lake/"], + )), + }, + GcsAccessTestCase { + description: "without one DuckDB has no way into the bucket, as it takes no service-account key", + hmac: None, + expected_access: None, + }, + ]; + for test_case in test_cases { + let store = StoreKind::Gcs { + bucket: "lake".to_owned(), + service_account_key: Some(Secret("{}".to_owned())), + endpoint: None, + hmac: test_case.hmac.map(|(key_id, secret)| HmacKey { + key_id: key_id.to_owned(), + secret: Secret(secret.to_owned()), + }), + }; + let access = bucket_access(&store).map_err(|error| error.to_string())?; + let found = access.map(|access| { + ( + access.bucket, + access.extensions, + access.statement, + access.parameters, + access.chain, + ) + }); + assert_eq!( + found, + test_case + .expected_access + .map(|(bucket, statement, parameters)| { + ( + bucket.to_owned(), + "INSTALL httpfs; LOAD httpfs;", + statement.to_owned(), + parameters + .iter() + .map(|value| (*value).to_owned()) + .collect::>(), + false, + ) + }), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/adapters/duckdb/tests/test_types.rs b/crates/sqlload/src/adapters/duckdb/tests/test_types.rs index ba8524835..fe7f8ffee 100644 --- a/crates/sqlload/src/adapters/duckdb/tests/test_types.rs +++ b/crates/sqlload/src/adapters/duckdb/tests/test_types.rs @@ -161,3 +161,11 @@ pub(crate) struct SecretCurrentTestCase { pub(crate) wanted: (&'static [&'static str], bool), pub(crate) expected_current: bool, } + +pub(crate) struct GcsAccessTestCase { + pub(crate) description: &'static str, + /// The connection's HMAC key ID and secret, if it has one. + pub(crate) hmac: Option<(&'static str, &'static str)>, + /// The secrets map key, the secret's statement and its parameters; `None` for no access. + pub(crate) expected_access: Option<(&'static str, &'static str, &'static [&'static str])>, +} diff --git a/crates/sqlload/src/adapters/files/main/folder_conflict.rs b/crates/sqlload/src/adapters/files/main/folder_conflict.rs index 6f0d47d26..02e1b0280 100644 --- a/crates/sqlload/src/adapters/files/main/folder_conflict.rs +++ b/crates/sqlload/src/adapters/files/main/folder_conflict.rs @@ -46,6 +46,6 @@ fn store_key(folder: &Location) -> String { }; format!("{}/{}", resolved.trim_end_matches('/'), folder.prefix) } - StoreKind::Sftp { .. } => folder.to_string(), + StoreKind::Gcs { .. } | StoreKind::Sftp { .. } => folder.to_string(), } } diff --git a/crates/sqlload/src/adapters/files/models.rs b/crates/sqlload/src/adapters/files/models.rs index fe8862a3d..3510fe0e1 100644 --- a/crates/sqlload/src/adapters/files/models.rs +++ b/crates/sqlload/src/adapters/files/models.rs @@ -19,7 +19,7 @@ use crate::store_paths::main::escape_partition_value::escape_partition_value; use crate::store_paths::main::unescape_partition_value::unescape_partition_value; use crate::store_paths::models::{Location, StoreKind}; -/// A target that writes tables as Parquet files into an s3 or filesystem connection. +/// A target that writes tables as Parquet files into an s3, gcs or filesystem connection. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FilesDestination { /// The connection's name, for messages and paths. diff --git a/crates/sqlload/src/file_store/_helpers/gcs_keys.rs b/crates/sqlload/src/file_store/_helpers/gcs_keys.rs new file mode 100644 index 000000000..1438a10b2 --- /dev/null +++ b/crates/sqlload/src/file_store/_helpers/gcs_keys.rs @@ -0,0 +1,83 @@ +//! The GCS client SQLLoad reads a bucket with, and where it gets its credentials: the connection's key, else Google's application default credentials. + +use object_store::ClientOptions; +use object_store::gcp::GoogleCloudStorageBuilder; + +/// Where a GCS client's credentials come from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum GcsCredentials { + /// The connection's `service_account_key`. + Connection, + /// The file `GOOGLE_APPLICATION_CREDENTIALS` names. + Named { path: String }, + /// The gcloud CLI's application default credentials file. + Gcloud { path: String }, + /// Neither: the service account of the machine SQLLoad runs on, from its metadata server. + Machine, +} + +impl GcsCredentials { + /// Where the credentials come from, for `sqlload debug`; never the credentials themselves. + pub(crate) fn describe(&self) -> String { + match self { + Self::Connection => "the connection's service_account_key".to_owned(), + Self::Named { path } => format!("GOOGLE_APPLICATION_CREDENTIALS ({path})"), + Self::Gcloud { path } => format!("gcloud's application default credentials ({path})"), + Self::Machine => "the machine's service account, from its metadata server".to_owned(), + } + } +} + +/// Where a GCS client's credentials come from, with `variable` reading the environment: the connection's key when it `has_key`; else the file `GOOGLE_APPLICATION_CREDENTIALS` names, which must exist; else gcloud's application default credentials file (`~/.config/gcloud/application_default_credentials.json`, or `%APPDATA%\gcloud\…` on Windows) when it exists; else the machine's service account. +pub(crate) fn gcs_credentials( + has_key: bool, + variable: &dyn Fn(&str) -> Option, +) -> Result { + if has_key { + return Ok(GcsCredentials::Connection); + } + if let Some(path) = variable("GOOGLE_APPLICATION_CREDENTIALS") { + return match std::path::Path::new(&path).is_file() { + true => Ok(GcsCredentials::Named { path }), + false => Err(format!( + "GOOGLE_APPLICATION_CREDENTIALS names {path}, which is not a file" + )), + }; + } + let (home, below) = match cfg!(windows) { + true => ("APPDATA", "gcloud/application_default_credentials.json"), + false => ( + "HOME", + ".config/gcloud/application_default_credentials.json", + ), + }; + Ok(variable(home) + .map(|home| std::path::Path::new(&home).join(below)) + .filter(|path| path.is_file()) + .map_or(GcsCredentials::Machine, |path| GcsCredentials::Gcloud { + path: path.display().to_string(), + })) +} + +/// The GCS client builder for `bucket`, signed in with the service-account key's JSON text `key`, else Google's application default credentials as [`gcs_credentials`] finds them with `variable` (the file `GOOGLE_APPLICATION_CREDENTIALS` names, gcloud's, else the machine's service account); the error when that variable names no file, rather than signing in as someone else. `endpoint` replaces Google's, as for an emulator. +pub(crate) fn gcs_builder( + bucket: &str, + key: Option<&str>, + endpoint: Option<&str>, + variable: &dyn Fn(&str) -> Option, +) -> Result { + let mut builder = GoogleCloudStorageBuilder::new().with_bucket_name(bucket); + match (key, gcs_credentials(key.is_some(), variable)?) { + (Some(key), _) => builder = builder.with_service_account_key(key), + (None, GcsCredentials::Named { path }) => { + builder = builder.with_application_credentials(path); + } + (None, _) => {} + } + if let Some(endpoint) = endpoint { + builder = builder.with_base_url(endpoint).with_client_options( + ClientOptions::new().with_allow_http(endpoint.starts_with("http://")), + ); + } + Ok(builder) +} diff --git a/crates/sqlload/src/file_store/_helpers/mod.rs b/crates/sqlload/src/file_store/_helpers/mod.rs index 160a40ded..991e01807 100644 --- a/crates/sqlload/src/file_store/_helpers/mod.rs +++ b/crates/sqlload/src/file_store/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod aws_keys; pub(crate) mod copies; +pub(crate) mod gcs_keys; pub(crate) mod keys; pub(crate) mod sftp; diff --git a/crates/sqlload/src/file_store/main/gcs_key_source.rs b/crates/sqlload/src/file_store/main/gcs_key_source.rs new file mode 100644 index 000000000..23a11af02 --- /dev/null +++ b/crates/sqlload/src/file_store/main/gcs_key_source.rs @@ -0,0 +1,9 @@ +//! Where a GCS connection's credentials come from, for `sqlload debug`. + +use crate::file_store::_helpers::aws_keys::environment_variable; +use crate::file_store::_helpers::gcs_keys::gcs_credentials; + +/// Where a GCS client's credentials come from (never the credentials themselves) when its connection `has_key` or not, or why they cannot be used. +pub fn gcs_key_source(has_key: bool) -> Result { + gcs_credentials(has_key, &environment_variable).map(|source| source.describe()) +} diff --git a/crates/sqlload/src/file_store/main/mod.rs b/crates/sqlload/src/file_store/main/mod.rs index df123e593..637bf826b 100644 --- a/crates/sqlload/src/file_store/main/mod.rs +++ b/crates/sqlload/src/file_store/main/mod.rs @@ -1,6 +1,7 @@ pub mod create_file; pub mod current_version; pub mod delete_files; +pub mod gcs_key_source; pub mod list_files; pub mod move_file; pub mod open_folder; diff --git a/crates/sqlload/src/file_store/main/open_folder.rs b/crates/sqlload/src/file_store/main/open_folder.rs index 3141e8f3f..1d5803f62 100644 --- a/crates/sqlload/src/file_store/main/open_folder.rs +++ b/crates/sqlload/src/file_store/main/open_folder.rs @@ -5,6 +5,9 @@ use std::sync::Arc; use object_store::ObjectStore; use object_store::local::LocalFileSystem; +use crate::auth::models::Secret; +use crate::file_store::_helpers::aws_keys::environment_variable; +use crate::file_store::_helpers::gcs_keys::gcs_builder; use crate::file_store::errors::StoreError; use crate::file_store::main::s3_builder::s3_builder; use crate::file_store::models::{OpenFolder, SftpStore}; @@ -12,7 +15,7 @@ use crate::http::main::install_tls_provider::install_tls_provider; use crate::store_paths::main::local_store_root::local_store_root; use crate::store_paths::models::{Location, StoreKind}; -/// A client for `location`'s store (an SFTP server is connected to on first use); S3 credentials come from `AWS_*` variables, the shared credentials file, web identity or the instance/container role. +/// A client for `location`'s store (an SFTP server is connected to on first use); S3 credentials come from `AWS_*` variables, the shared credentials file, web identity or the instance/container role; GCS credentials from the connection's key, else Google's application default credentials. pub fn open_folder(location: &Location) -> Result { let failed = |source: object_store::Error| StoreError::Open { location: location.to_string(), @@ -41,6 +44,27 @@ pub fn open_folder(location: &Location) -> Result { .build() .map_err(failed)?, ), + StoreKind::Gcs { + bucket, + service_account_key, + endpoint, + .. + } => Arc::new( + gcs_builder( + bucket, + service_account_key.as_ref().map(Secret::expose), + endpoint.as_deref(), + &environment_variable, + ) + .map_err(|reason| { + failed(object_store::Error::Generic { + store: "GCS", + source: reason.into(), + }) + })? + .build() + .map_err(failed)?, + ), StoreKind::Filesystem { root } => { std::fs::create_dir_all(root).map_err(|error| { failed(object_store::Error::Generic { diff --git a/crates/sqlload/src/file_store/tests.rs b/crates/sqlload/src/file_store/tests.rs index b714a12e8..aead03932 100644 --- a/crates/sqlload/src/file_store/tests.rs +++ b/crates/sqlload/src/file_store/tests.rs @@ -1,4 +1,6 @@ #[path = "tests/test_aws_keys.rs"] mod aws_keys; +#[path = "tests/test_gcs_keys.rs"] +mod gcs_keys; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlload/src/file_store/tests/test_gcs_keys.rs b/crates/sqlload/src/file_store/tests/test_gcs_keys.rs new file mode 100644 index 000000000..13d03104f --- /dev/null +++ b/crates/sqlload/src/file_store/tests/test_gcs_keys.rs @@ -0,0 +1,87 @@ +use crate::file_store::_helpers::gcs_keys::{gcs_builder, gcs_credentials}; +use crate::file_store::tests::test_types::GcsCredentialsTestCase; + +#[test] +fn given_a_gcs_connection_with_or_without_a_key_when_choosing_its_credentials_then_the_first_in_googles_chain_wins_and_a_missing_named_file_fails_the_client() +-> Result<(), String> { + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + let home = directory.path().join("home"); + for gcloud in [home.join(".config/gcloud"), home.join("gcloud")] { + std::fs::create_dir_all(&gcloud).map_err(|error| error.to_string())?; + std::fs::write(gcloud.join("application_default_credentials.json"), "{}") + .map_err(|error| error.to_string())?; + } + std::fs::write(directory.path().join("sa.json"), "{}").map_err(|error| error.to_string())?; + let test_cases = [ + GcsCredentialsTestCase { + description: "the connection's key wins over the environment", + has_key: true, + named: Some("sa.json"), + home: Some("home"), + expected_text: "the connection's service_account_key", + expected_builds: true, + }, + GcsCredentialsTestCase { + description: "GOOGLE_APPLICATION_CREDENTIALS wins over gcloud's file", + has_key: false, + named: Some("sa.json"), + home: Some("home"), + expected_text: "GOOGLE_APPLICATION_CREDENTIALS (", + expected_builds: true, + }, + GcsCredentialsTestCase { + description: "GOOGLE_APPLICATION_CREDENTIALS naming no file fails rather than falling through", + has_key: false, + named: Some("gone.json"), + home: Some("home"), + expected_text: "gone.json, which is not a file", + expected_builds: false, + }, + GcsCredentialsTestCase { + description: "without either, gcloud's application default credentials", + has_key: false, + named: None, + home: Some("home"), + expected_text: "gcloud's application default credentials (", + expected_builds: true, + }, + GcsCredentialsTestCase { + description: "with none of them, the machine's service account", + has_key: false, + named: None, + home: Some("nobody"), + expected_text: "the machine's service account", + expected_builds: true, + }, + ]; + for test_case in &test_cases { + let variables = [ + ("GOOGLE_APPLICATION_CREDENTIALS", test_case.named), + ("HOME", test_case.home), + ("APPDATA", test_case.home), + ]; + let variable = |name: &str| { + variables + .iter() + .find(|(variable, _)| *variable == name) + .and_then(|(_, relative)| *relative) + .map(|relative| directory.path().join(relative).display().to_string()) + }; + let found = gcs_credentials(test_case.has_key, &variable) + .map(|source| source.describe()) + .unwrap_or_else(|error| error); + assert!( + found.contains(test_case.expected_text), + "{}: {found}", + test_case.description + ); + let built = gcs_builder("lake", test_case.has_key.then_some("{}"), None, &variable); + assert_eq!( + built.is_ok(), + test_case.expected_builds, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/file_store/tests/test_types.rs b/crates/sqlload/src/file_store/tests/test_types.rs index 2750e45b0..f9d810e68 100644 --- a/crates/sqlload/src/file_store/tests/test_types.rs +++ b/crates/sqlload/src/file_store/tests/test_types.rs @@ -27,3 +27,17 @@ pub(crate) struct KeyedBuilderTestCase { /// The client's key ID, session token and region. pub(crate) expected_config: [Option<&'static str>; 3], } + +pub(crate) struct GcsCredentialsTestCase { + pub(crate) description: &'static str, + /// Whether the connection sets `service_account_key`. + pub(crate) has_key: bool, + /// `GOOGLE_APPLICATION_CREDENTIALS`, in the test's folder: `sa.json` exists, `gone.json` doesn't. + pub(crate) named: Option<&'static str>, + /// The home folder, in the test's folder: `home` holds gcloud's file, `nobody` doesn't exist. + pub(crate) home: Option<&'static str>, + /// Text where the credentials come from, or the error, must contain. + pub(crate) expected_text: &'static str, + /// Whether the client builder takes those credentials, rather than failing with the error. + pub(crate) expected_builds: bool, +} diff --git a/crates/sqlload/src/loader/_helpers/python.rs b/crates/sqlload/src/loader/_helpers/python.rs index 7f9aa7d5b..597f050ff 100644 --- a/crates/sqlload/src/loader/_helpers/python.rs +++ b/crates/sqlload/src/loader/_helpers/python.rs @@ -73,7 +73,7 @@ use crate::number_text::main::thousands::thousands; use crate::postgres_source::models::PgType; use crate::postgres_source::models::SourceColumn; use crate::postgres_source::models::SourceTable; -use crate::project::constants::FILES_CONNECTION_KINDS; +use crate::project::constants::SHAREABLE_CONNECTION_KINDS; use crate::project::models::CheckName; use crate::project::models::Contract; use crate::project::models::NullCursor; @@ -538,12 +538,12 @@ pub(crate) async fn refuse_emptying( } } -/// Whether a target's connection `name` may also be a Python source's own: only an s3 or filesystem one (project checks refuse the rest). +/// Whether a target's connection `name` may also be a Python source's own: only an s3 or filesystem one, which holds no credentials (project checks refuse the rest). pub(crate) fn shareable_connection(project: &ProjectConfig, name: &str) -> bool { project .connections .get(name) - .is_some_and(|connection| FILES_CONNECTION_KINDS.contains(&connection.kind())) + .is_some_and(|connection| SHAREABLE_CONNECTION_KINDS.contains(&connection.kind())) } /// The largest non-NULL value of cursor `column` (matched ignoring case) in `batch`, as cursor text; `None` when it has none. diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index cad82faad..648fd23bc 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -45,7 +45,8 @@ use crate::number_text::main::thousands::thousands; use crate::project::models::BatchStep; use crate::project::models::MaskMethod; use crate::project::models::{FilesFormat, OutputCompression}; -use crate::routing::models::{Route, Step, Store}; +use crate::routing::constants::LOADER_READS_BECAUSE; +use crate::routing::models::{FileFormat, Route, Step, Store}; use crate::row_batches::main::flatten_lines::flatten_lines; use crate::row_batches::models::Unnest; use crate::run_log::main::describe_state::describe_state; @@ -677,10 +678,18 @@ pub(crate) fn load_item(table: &TablePlan) -> PlanItem { "changed columns move to the end of the column order", )); } - let why_shown = + let staged_why = (table.stage.is_some() && spec.files.is_some() && !is_direct(table.route.as_ref())) || spec.warehouse_source.is_some(); - let archived = archived_reason(spec, table.files.as_ref()).filter(|_| !why_shown); + let archived = archived_reason(spec, table.files.as_ref()).filter(|_| !staged_why); + let why_shown = staged_why + || table.route.as_ref().is_some_and(|route| { + route.reason.starts_with(LOADER_READS_BECAUSE) + && archived.as_ref().is_none_or(|note| !route.reason.contains(note)) + && route.steps.iter().any(|step| { + matches!(step, Step::UseFiles(files) if files.format == FileFormat::Parquet) + }) + }); details.extend(table.route.as_ref().map(|route| { let text = route_text(table, route); detail( diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index da2a69cf9..c20413ecd 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -129,9 +129,9 @@ use crate::snowflake_source::main::read_catalog::read_catalog as read_snowflake_ use crate::snowflake_source::main::unload_files::unload_files; use crate::snowflake_source::main::unload_rows::unload_rows; use crate::snowflake_source::models::{UnloadRequest, UnloadedFiles}; -use crate::ssh_tunnel::models::SshTunnel; +use crate::store_paths::main::connection_store::connection_store; use crate::store_paths::main::resolve_location::resolve_location; -use crate::store_paths::models::{Location, StoreKind}; +use crate::store_paths::models::Location; /// The sources the plan running now has finished checking, which only the line `plan` shows on a terminal reads. pub(crate) static PLANNED_SOURCES: std::sync::atomic::AtomicUsize = @@ -151,45 +151,15 @@ pub(crate) fn files_location( project: &ProjectConfig, source: &SourceConfig, ) -> Result { - let store = match project.connections.get(&source.connection) { - Some(ConnectionConfig::S3 { - bucket, - region, - endpoint, - profile, - }) => StoreKind::S3 { - bucket: bucket.clone(), - region: region.clone(), - endpoint: endpoint.clone(), - profile: profile.clone(), - }, - Some(ConnectionConfig::Filesystem { root }) => StoreKind::Filesystem { root: root.clone() }, - Some(ConnectionConfig::Sftp { - host, - port, - user, - private_key_file, - private_key_passphrase, - password, - known_hosts, - }) => StoreKind::Sftp { - server: Box::new(SshTunnel { - host: host.clone(), - port: *port, - user: user.clone(), - private_key_file: private_key_file.clone(), - private_key_passphrase: private_key_passphrase.clone(), - password: password.clone(), - known_hosts: Some(known_hosts.clone()), - }), - root: String::new(), - }, - _ => { - return Err(LoaderError::Invalid(format!( - "connection {:?} is not an s3, filesystem or sftp connection", - source.connection - ))); - } + let Some(store) = project + .connections + .get(&source.connection) + .and_then(connection_store) + else { + return Err(LoaderError::Invalid(format!( + "connection {:?} is not an s3, gcs, filesystem or sftp connection", + source.connection + ))); }; let path = source.path.as_deref().unwrap_or_default(); resolve_location(Some((source.connection.as_str(), &store)), path) diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index d285ecade..11cd8fa8d 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -999,7 +999,7 @@ pub(crate) fn reader_offer( ) -> ReaderOffer { let existing_files = spec.files.as_ref().map(|files| FileSet { store: match files.location.store { - StoreKind::S3 { .. } | StoreKind::Sftp { .. } => { + StoreKind::S3 { .. } | StoreKind::Gcs { .. } | StoreKind::Sftp { .. } => { Store::ObjectStore(files.location.to_string()) } StoreKind::Filesystem { .. } => Store::Local(files.location.to_string()), diff --git a/crates/sqlload/src/loader/main/debug_project.rs b/crates/sqlload/src/loader/main/debug_project.rs index 336917c98..d99201e4f 100644 --- a/crates/sqlload/src/loader/main/debug_project.rs +++ b/crates/sqlload/src/loader/main/debug_project.rs @@ -2,11 +2,12 @@ use crate::adapters::dispatch::main::session::check_destination::check_destination; use crate::adapters::dispatch::main::session::connect_destination::connect_destination; +use crate::file_store::main::gcs_key_source::gcs_key_source; use crate::file_store::main::s3_key_source::s3_key_source; use crate::loader::main::_check_source::check_source; use crate::project::models::{ConnectionConfig, ProjectConfig}; -/// One line per check (`ok`, `warn` or `fail`), and whether nothing failed: where each S3 connection's keys come from, then each target and source; `connect = false` skips them. +/// One line per check (`ok`, `warn` or `fail`), and whether nothing failed: where each S3 connection's keys and each GCS connection's credentials come from, then each target and source; `connect = false` skips them. pub async fn debug_project(project: &ProjectConfig, connect: bool) -> (Vec, bool) { let mut lines = vec![format!( "ok config {} connections, {} targets, {} sources", @@ -32,11 +33,19 @@ pub async fn debug_project(project: &ProjectConfig, connect: bool) -> (Vec { + s3_key_source(profile.as_deref()).map(|source| format!("S3 keys from {source}")) + } + ConnectionConfig::Gcs { + service_account_key, + .. + } => gcs_key_source(service_account_key.is_some()) + .map(|source| format!("GCS credentials from {source}")), + _ => continue, }; - lines.push(match s3_key_source(profile.as_deref()) { - Ok(source) => format!("ok connection {name} S3 keys from {source}"), + lines.push(match found { + Ok(source) => format!("ok connection {name} {source}"), Err(error) => { ok = false; format!("fail connection {name} {error}") diff --git a/crates/sqlload/src/loader/tests/helpers.rs b/crates/sqlload/src/loader/tests/helpers.rs index 323ecc23a..2d2e0c94e 100644 --- a/crates/sqlload/src/loader/tests/helpers.rs +++ b/crates/sqlload/src/loader/tests/helpers.rs @@ -6,9 +6,10 @@ use arrow_cast::display::{ArrayFormatter, FormatOptions}; use arrow_schema::{DataType, Field, Schema, TimeUnit}; +use crate::auth::models::Secret; use crate::column_types::constants::ARROW_TYPE_METADATA_KEY; use crate::file_store::errors::StoreError; -use crate::files_source::models::{FilesEncoding, FilesTable, TextChecks}; +use crate::files_source::models::{FilesEncoding, FilesRead, FilesTable, TextChecks}; use crate::http::errors::ApiError; use crate::loader::_helpers::python::child_paths; use crate::loader::_helpers::python::{ @@ -19,15 +20,16 @@ use crate::loader::_helpers::table_plan::reader_offer; use crate::loader::errors::LoaderError; use crate::loader::models::{FilesSource, PlanDetail, PlanItem, TableSpec, TableTransform}; use crate::loader::tests::test_types::{ - ChildPathsTestCase, DeclaredProblemsTestCase, InferredTableTestCase, ItemSpec, - OlderFilesTestCase, ReloadOrderTestCase, SourceSpellingTestCase, TransformReaderTestCase, + ChildPathsTestCase, DeclaredProblemsTestCase, GcsReaderTestCase, InferredTableTestCase, + ItemSpec, OlderFilesTestCase, ReloadOrderTestCase, SourceSpellingTestCase, + TransformReaderTestCase, }; use crate::postgres_source::models::{PgType, SourceColumn, SourceTable}; use crate::project::main::parse_python_project::parse_python_project; use crate::python_sources::models::PythonSource; use crate::routing::models::{FileFormat, FileSet, Route, Step, Store}; use crate::schema_plan::models::LiveColumn; -use crate::store_paths::models::{Location, StoreKind}; +use crate::store_paths::models::{HmacKey, Location, StoreKind}; /// The columns of the inferred table, as `name:type` joined by commas. pub(crate) fn inferred_columns(test_case: &InferredTableTestCase) -> String { @@ -540,13 +542,50 @@ pub(crate) fn not_null_keys( pub(crate) fn transform_reader( test_case: &TransformReaderTestCase, ) -> Result, String> { + let mut spec = files_spec(StoreKind::Filesystem { + root: "/data".to_owned(), + })?; + spec.transform = test_case.transform.map(|function| TableTransform { + function: function.to_owned(), + version: format!("{function}@0"), + older: 0, + creates_log: false, + }); + let parse = column_parse(&serde_json::json!({ "timestamp": ["iso"] }))?; + spec.parses + .extend(test_case.parse.map(|column| (column.to_owned(), parse))); + Ok(reader_offer(&spec, None, None).needs_loader) +} + +/// Why DuckDB can't read a Parquet files table's files in GCS itself, with the case's HMAC key and endpoint on the connection. +pub(crate) fn gcs_reader(test_case: &GcsReaderTestCase) -> Result, String> { + let spec = files_spec(StoreKind::Gcs { + bucket: "lake".to_owned(), + service_account_key: None, + endpoint: test_case.endpoint.map(str::to_owned), + hmac: test_case.hmac.then(|| HmacKey { + key_id: "GOOG1EXAMPLE".to_owned(), + secret: Secret("hm4c".to_owned()), + }), + })?; + let read = FilesRead { + files: Vec::new(), + encoding: FilesEncoding::default(), + checks: TextChecks::default(), + columns: Vec::new(), + nanos: Vec::new(), + batch_rows: 1000, + }; + Ok(reader_offer(&spec, Some(&read), None).direct_blocked) +} + +/// A Parquet files table in `drop/` of `store`, loading into DuckDB. +fn files_spec(store: StoreKind) -> Result { let mut spec = python_spec("{}")?; spec.python = None; spec.files = Some(FilesSource { location: Location { - store: StoreKind::Filesystem { - root: "/data".to_owned(), - }, + store, prefix: "drop/".to_owned(), }, table: FilesTable { @@ -568,16 +607,7 @@ pub(crate) fn transform_reader( transform: None, archive: None, }); - spec.transform = test_case.transform.map(|function| TableTransform { - function: function.to_owned(), - version: format!("{function}@0"), - older: 0, - creates_log: false, - }); - let parse = column_parse(&serde_json::json!({ "timestamp": ["iso"] }))?; - spec.parses - .extend(test_case.parse.map(|column| (column.to_owned(), parse))); - Ok(reader_offer(&spec, None, None).needs_loader) + Ok(spec) } /// How many files `older_files` counts in the case's records, for table `t` of a Python source, whose `transform` is now at `f@2`. diff --git a/crates/sqlload/src/loader/tests/test_transform_reader.rs b/crates/sqlload/src/loader/tests/test_transform_reader.rs index cfedc92d9..fe29989e2 100644 --- a/crates/sqlload/src/loader/tests/test_transform_reader.rs +++ b/crates/sqlload/src/loader/tests/test_transform_reader.rs @@ -1,5 +1,5 @@ -use crate::loader::tests::helpers::transform_reader; -use crate::loader::tests::test_types::TransformReaderTestCase; +use crate::loader::tests::helpers::{gcs_reader, transform_reader}; +use crate::loader::tests::test_types::{GcsReaderTestCase, TransformReaderTestCase}; #[test] fn given_a_files_table_with_a_transform_or_a_parse_when_routing_then_sqlload_reads_the_files_itself() @@ -34,3 +34,39 @@ fn given_a_files_table_with_a_transform_or_a_parse_when_routing_then_sqlload_rea } Ok(()) } + +#[test] +fn given_parquet_files_in_gcs_when_routing_into_duckdb_then_duckdb_reads_them_only_with_the_connections_hmac_key() +-> Result<(), String> { + let test_cases = [ + GcsReaderTestCase { + description: "with an HMAC key DuckDB reads the bucket itself", + hmac: true, + endpoint: None, + expected_reason: None, + }, + GcsReaderTestCase { + description: "without one SQLLoad reads the files, as DuckDB takes no service-account key", + hmac: false, + endpoint: None, + expected_reason: Some( + "the gcs connection sets no hmac_key_id, which DuckDB and MotherDuck read GCS with", + ), + }, + GcsReaderTestCase { + description: "an emulator's endpoint is SQLLoad's to follow, not DuckDB's", + hmac: true, + endpoint: Some("http://127.0.0.1:4443"), + expected_reason: Some("the GCS connection sets a custom endpoint"), + }, + ]; + for test_case in test_cases { + assert_eq!( + gcs_reader(&test_case)?.as_deref(), + test_case.expected_reason, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs index 4a9e43bb7..1d930bf5b 100644 --- a/crates/sqlload/src/loader/tests/test_types.rs +++ b/crates/sqlload/src/loader/tests/test_types.rs @@ -264,3 +264,13 @@ pub(crate) struct MostWindowsTestCase { pub(crate) spec: fn() -> Result, pub(crate) expected_most: Result, } + +pub(crate) struct GcsReaderTestCase { + pub(crate) description: &'static str, + /// Whether the gcs connection sets an HMAC key. + pub(crate) hmac: bool, + /// The gcs connection's custom endpoint, if any. + pub(crate) endpoint: Option<&'static str>, + /// Why SQLLoad must read the files instead of DuckDB, if it must. + pub(crate) expected_reason: Option<&'static str>, +} diff --git a/crates/sqlload/src/loader/tests/test_view_lines.rs b/crates/sqlload/src/loader/tests/test_view_lines.rs index 221d6dae1..503c86d89 100644 --- a/crates/sqlload/src/loader/tests/test_view_lines.rs +++ b/crates/sqlload/src/loader/tests/test_view_lines.rs @@ -198,7 +198,7 @@ fn given_tables_stopped_for_the_same_reasons_when_printing_errors_then_they_shar } #[test] -fn given_tables_on_their_destinations_when_printing_the_plan_then_each_destination_is_named_and_each_route_names_it_when_there_are_several() +fn given_tables_on_their_destinations_when_printing_the_plan_then_each_destination_is_named_and_each_route_names_it_and_one_table_says_why_sqlload_reads_parquet() { let test_cases = [ RouteLinesTestCase { @@ -221,6 +221,18 @@ fn given_tables_on_their_destinations_when_printing_the_plan_then_each_destinati "route read parquet into Arrow -> DuckDB takes Arrow", ], }, + RouteLinesTestCase { + description: "a plan of one Parquet table the destination could read itself says why SQLLoad reads it", + tables: &[RoutedTable { + name: "b", + target: "dev", + route: masked_files_route, + }], + expected_lines: &[ + "route read parquet into Arrow -> DuckDB takes Arrow", + "why SQLLoad reads the files because email is masked with hmac", + ], + }, RouteLinesTestCase { description: "two destinations: one to line each, and every route names its destination", tables: &[ @@ -264,7 +276,9 @@ fn given_tables_on_their_destinations_when_printing_the_plan_then_each_destinati }; let lines: Vec = view_lines(&text_view(&plan)) .into_iter() - .filter(|line| line.starts_with("to ") || line.contains("── route")) + .filter(|line| { + line.starts_with("to ") || line.contains("── route") || line.contains("── why") + }) .map(|line| { line.trim_start_matches(['│', '├', '└', '─', ' ']) .to_owned() diff --git a/crates/sqlload/src/project/_helpers/references.rs b/crates/sqlload/src/project/_helpers/references.rs index 53acfba9a..b5b862fcd 100644 --- a/crates/sqlload/src/project/_helpers/references.rs +++ b/crates/sqlload/src/project/_helpers/references.rs @@ -11,7 +11,7 @@ use crate::project::_helpers::targets::{ }; use crate::project::_helpers::tombstones::tombstone_problems; use crate::project::constants::{ - FILES_CONNECTION_KINDS, MAX_BATCH_CONCURRENCY, MAX_POSTGRES_NAME_BYTES, MIN_BATCH_INTERVAL, + MAX_BATCH_CONCURRENCY, MAX_POSTGRES_NAME_BYTES, MIN_BATCH_INTERVAL, SHAREABLE_CONNECTION_KINDS, STAGE_NAME_PARTS, TABLE_PLACEHOLDER, }; use crate::project::models::{ @@ -52,6 +52,17 @@ pub(crate) fn reference_problems(project: &ProjectConfig) -> Vec { "[connections.{name}] profile is empty; omit it to use AWS_PROFILE, else default" )); } + if let ConnectionConfig::Gcs { + hmac_key_id, + hmac_secret, + .. + } = connection + && hmac_key_id.is_some() != hmac_secret.is_some() + { + problems.push(format!( + "[connections.{name}] hmac_key_id and hmac_secret go together: set both for DuckDB and MotherDuck to read the bucket themselves, or neither" + )); + } } for (name, source) in &project.sources { problems.extend(source_problems(project, name, source)); @@ -108,7 +119,10 @@ fn source_problems(project: &ProjectConfig, name: &str, source: &SourceConfig) - SourceKind::Databricks => (&["databricks"], "databricks"), SourceKind::DuckDb => (&["duckdb"], "duckdb"), SourceKind::MotherDuck => (&["motherduck"], "motherduck"), - SourceKind::Files => (&["s3", "filesystem", "sftp"], "s3, filesystem or sftp"), + SourceKind::Files => ( + &["s3", "gcs", "filesystem", "sftp"], + "s3, gcs, filesystem or sftp", + ), SourceKind::Python => (&[], "any"), SourceKind::GoogleSheets => (&["google_sheets"], "google_sheets"), }; @@ -175,14 +189,14 @@ fn source_problems(project: &ProjectConfig, name: &str, source: &SourceConfig) - problems } -/// A Python source reading a connection a warehouse target loads into, whose credentials Python code must never get (a files target's s3 or filesystem connection may be shared). +/// A Python source reading a connection a warehouse target loads into, whose credentials Python code must never get (a files target's s3 or filesystem connection, which holds no credentials, may be shared). fn shared_warehouse_problems( project: &ProjectConfig, name: &str, source: &SourceConfig, connection: &ConnectionConfig, ) -> Vec { - if FILES_CONNECTION_KINDS.contains(&connection.kind()) { + if SHAREABLE_CONNECTION_KINDS.contains(&connection.kind()) { return Vec::new(); } project diff --git a/crates/sqlload/src/project/_helpers/targets.rs b/crates/sqlload/src/project/_helpers/targets.rs index a7f4356d7..0cd2503e6 100644 --- a/crates/sqlload/src/project/_helpers/targets.rs +++ b/crates/sqlload/src/project/_helpers/targets.rs @@ -78,7 +78,7 @@ pub(crate) fn stage_problems( problems } -/// Whether target `name` writes files (its connection is s3 or filesystem). +/// Whether target `name` writes files (its connection is s3, gcs or filesystem). pub(crate) fn is_files_target(project: &ProjectConfig, name: &str) -> bool { project .targets @@ -145,7 +145,7 @@ pub(crate) fn target_problems( } if files_settings { problems.push(format!( - "{place} path, format, compression, csv, single_file and max_footer_reads are for targets on an s3 or filesystem connection" + "{place} path, format, compression, csv, single_file and max_footer_reads are for targets on an s3, gcs or filesystem connection" )); } } @@ -164,7 +164,10 @@ pub(crate) fn target_problems( "{place} connection {:?} is a {kind} connection; targets load into {} or write files to {}", target.connection, WAREHOUSE_CONNECTION_KINDS.join(", "), - FILES_CONNECTION_KINDS.join(" or ") + FILES_CONNECTION_KINDS + .split_last() + .map(|(last, rest)| format!("{} or {last}", rest.join(", "))) + .unwrap_or_default() )), } problems @@ -184,7 +187,7 @@ pub(crate) fn destination_kind_problems( let mut problems: Vec = Vec::new(); if sets_path && !files_target { problems.push(format!( - "{place} destination_path is for targets on an s3 or filesystem connection" + "{place} destination_path is for targets on an s3, gcs or filesystem connection" )); } if source.schema.is_some() && files_target { diff --git a/crates/sqlload/src/project/constants.rs b/crates/sqlload/src/project/constants.rs index b7733f691..b7df53288 100644 --- a/crates/sqlload/src/project/constants.rs +++ b/crates/sqlload/src/project/constants.rs @@ -155,7 +155,10 @@ pub const WAREHOUSE_CONNECTION_KINDS: &[&str] = &[ pub const CLICKHOUSE_CONNECTION_KIND: &str = "clickhouse"; /// The connection kinds a target writes files into. -pub const FILES_CONNECTION_KINDS: &[&str] = &["s3", "filesystem"]; +pub const FILES_CONNECTION_KINDS: &[&str] = &["s3", "gcs", "filesystem"]; + +/// The files connection kinds a Python source may share with a files target, as they hold no credentials (a gcs connection holds the bucket's keys). +pub const SHAREABLE_CONNECTION_KINDS: &[&str] = &["s3", "filesystem"]; /// The table folder template of a files target that sets no `path`. pub const DEFAULT_DESTINATION_PATH: &str = "{source}/{table}/"; diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index 549cc5984..a63f72be9 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -224,6 +224,22 @@ pub enum ConnectionConfig { #[serde(default)] profile: Option, }, + /// A Google Cloud Storage bucket that files sources read from; credentials come from `service_account_key`, else Google's application default credentials (`GOOGLE_APPLICATION_CREDENTIALS`, the gcloud user's, or the instance's service account). + Gcs { + bucket: String, + /// The service-account key file's JSON text, such as `${ENV:GCS_KEY}`. + #[serde(default)] + service_account_key: Option, + /// A custom endpoint, for tests. + #[serde(default)] + endpoint: Option, + /// The access ID of an HMAC key for the bucket, which DuckDB and MotherDuck need to read its Parquet files themselves (they take no service-account key). + #[serde(default)] + hmac_key_id: Option, + /// That HMAC key's secret, such as `${ENV:GCS_HMAC_SECRET}`. + #[serde(default)] + hmac_secret: Option, + }, /// A directory on the machine SQLLoad runs on. Filesystem { root: String }, /// An SFTP server that files sources read from; its host key must be listed in `known_hosts`. @@ -330,6 +346,7 @@ impl ConnectionConfig { Self::DuckDb { .. } => "duckdb", Self::MotherDuck { .. } => "motherduck", Self::S3 { .. } => "s3", + Self::Gcs { .. } => "gcs", Self::Filesystem { .. } => "filesystem", Self::Sftp { .. } => "sftp", Self::Odbc { .. } => "odbc", @@ -339,7 +356,7 @@ impl ConnectionConfig { } } -/// Where loaded tables go in one environment: a warehouse, or files in an s3 or filesystem connection. +/// Where loaded tables go in one environment: a warehouse, or files in an s3, gcs or filesystem connection. #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct TargetConfig { diff --git a/crates/sqlload/src/project/tests/test_parse_project.rs b/crates/sqlload/src/project/tests/test_parse_project.rs index 3bc750fe0..679801b76 100644 --- a/crates/sqlload/src/project/tests/test_parse_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_project.rs @@ -312,7 +312,7 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { ParseProjectErrorTestCase { description: "a warehouse target takes no path", text: "[targets.other]\nconnection = \"wh\"\nloader_schema = \"RAW\"\npath = \"{table}/\"\n", - expected_message: "[targets.other] path, format, compression, csv, single_file and max_footer_reads are for targets on an s3 or filesystem connection", + expected_message: "[targets.other] path, format, compression, csv, single_file and max_footer_reads are for targets on an s3, gcs or filesystem connection", }, ParseProjectErrorTestCase { description: "a files target takes no loader_schema", @@ -479,6 +479,11 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { text: "[connections.lake]\nkind = \"s3\"\nbucket = \"b\"\nprofile = \"\"\n", expected_message: "[connections.lake] profile is empty; omit it", }, + ParseProjectErrorTestCase { + description: "an HMAC key's ID without its secret signs nothing", + text: "[connections.lake]\nkind = \"gcs\"\nbucket = \"b\"\nhmac_key_id = \"GOOG1E\"\n", + expected_message: "[connections.lake] hmac_key_id and hmac_secret go together", + }, ParseProjectErrorTestCase { description: "an odbc connection needs a connection string or a DSN", text: "[connections.erp]\nkind = \"odbc\"\n", @@ -617,17 +622,17 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { ParseProjectErrorTestCase { description: "a target cannot load into an HTTP API", text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\n[targets.other]\nconnection = \"api\"\nloader_schema = \"RAW\"\n", - expected_message: "is a http connection; targets load into snowflake, databricks, bigquery, clickhouse, duckdb, motherduck, postgres or write files to s3 or filesystem", + expected_message: "is a http connection; targets load into snowflake, databricks, bigquery, clickhouse, duckdb, motherduck, postgres or write files to s3, gcs or filesystem", }, ParseProjectErrorTestCase { description: "destination_path needs a files target", text: "[sources.app]\nkind = \"postgres\"\nconnection = \"pg\"\ninclude = [\"*\"]\ndestination_path = \"{table}/\"\n", - expected_message: "[sources.app] destination_path is for targets on an s3 or filesystem connection", + expected_message: "[sources.app] destination_path is for targets on an s3, gcs or filesystem connection", }, ParseProjectErrorTestCase { description: "a table's destination_path needs a files target too", text: "[sources.app]\nkind = \"postgres\"\nconnection = \"pg\"\ninclude = [\"*\"]\n[sources.app.tables.\"public.a\"]\ndestination_path = \"{table}/\"\n", - expected_message: "[sources.app] destination_path is for targets on an s3 or filesystem connection", + expected_message: "[sources.app] destination_path is for targets on an s3, gcs or filesystem connection", }, ParseProjectErrorTestCase { description: "a source schema is for warehouse targets", diff --git a/crates/sqlload/src/project/tests/test_parse_python_project.rs b/crates/sqlload/src/project/tests/test_parse_python_project.rs index e0fdb684e..da656e1e5 100644 --- a/crates/sqlload/src/project/tests/test_parse_python_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_python_project.rs @@ -102,6 +102,15 @@ fn given_python_sources_when_parsing_the_project_then_they_join_it_under_the_sam read: |project| project.sources["shop"].connection.clone(), expected_value: Ok("lake"), }, + ParsePythonProjectTestCase { + description: "a Python source may not read the gcs connection a files target writes to, as it holds the bucket's keys", + toml: "[connections.lake]\nkind = \"gcs\"\nbucket = \"lake\"\nservice_account_key = \"{}\"\n[targets.files]\nconnection = \"lake\"\n", + sources: r#"[{"name": "shop", "module": "sources/shop.py", "settings": {"connection": "lake", "destination": "dev", "tables": {}}}]"#, + read: |project| project.sources["shop"].connection.clone(), + expected_value: Err( + "Python source shop uses connection lake, which target files loads into; Python code never gets warehouse credentials — give the source its own connection", + ), + }, ParsePythonProjectTestCase { description: "a None setting from Python is refused", toml: "", diff --git a/crates/sqlload/src/routing/_helpers/files_route.rs b/crates/sqlload/src/routing/_helpers/files_route.rs index c5291102b..5f9e6d083 100644 --- a/crates/sqlload/src/routing/_helpers/files_route.rs +++ b/crates/sqlload/src/routing/_helpers/files_route.rs @@ -49,7 +49,7 @@ pub(crate) fn warehouse_reads( } } -/// Files for a destination that does not read them itself: in place where it can, otherwise read by SQLLoad, except files it reads only itself ([`WriterAccepts::must_pull`]), which fail. +/// Files for a destination that does not read them itself: in place where it can, otherwise read by SQLLoad, except files it reads only itself ([`WriterAccepts::must_pull`]), which fail; the reason names what keeps the destination from reading them only when it could have. pub(crate) fn other_reads( reader: &ReaderOffer, writer: &WriterAccepts, @@ -65,23 +65,27 @@ pub(crate) fn other_reads( ], reason: "the files already exist and the destination reads them itself".to_owned(), }), - (_, Some(reason), _) | (true, None, Some(reason)) => loader_reads( - files, - writer, - overrides, - &format!("{LOADER_READS_BECAUSE} {reason}"), - ), - (false, None, Some(reason)) if must_pull(writer, files).is_some() => loader_reads( + (true, Some(reason), _) | (true, None, Some(reason)) => loader_reads( files, writer, overrides, &format!("{LOADER_READS_BECAUSE} {reason}"), ), + (false, Some(reason), _) | (false, None, Some(reason)) + if must_pull(writer, files).is_some() => + { + loader_reads( + files, + writer, + overrides, + &format!("{LOADER_READS_BECAUSE} {reason}"), + ) + } (false, None, None) if must_pull(writer, files).is_some() => Err(RouteError::Uncovered { location: files.store.to_string(), reason: must_pull(writer, files).unwrap_or_default(), }), - (false, None, _) => loader_reads( + (false, _, _) => loader_reads( files, writer, overrides, diff --git a/crates/sqlload/src/routing/tests/test_plan_route.rs b/crates/sqlload/src/routing/tests/test_plan_route.rs index 135e79caa..2c6dd8755 100644 --- a/crates/sqlload/src/routing/tests/test_plan_route.rs +++ b/crates/sqlload/src/routing/tests/test_plan_route.rs @@ -420,15 +420,35 @@ fn given_an_impossible_combination_when_planning_then_the_error_says_why() { #[test] fn given_files_sqlload_must_read_when_planning_then_the_route_says_why() { - let test_cases = [PlanRouteReasonTestCase { - description: "Parquet in S3 Databricks can't load exactly, outside every external location", - reader: ReaderOffer { - direct_blocked: Some("at is a time".to_owned()), - ..files_source("s3://other/scores/", FileFormat::Parquet) - }, - writer: databricks_destination(), - expected_reason: "SQLLoad reads the files because at is a time", - }]; + let test_cases = [ + PlanRouteReasonTestCase { + description: "Parquet in S3 Databricks can't load exactly, outside every external location", + reader: ReaderOffer { + direct_blocked: Some("at is a time".to_owned()), + ..files_source("s3://other/scores/", FileFormat::Parquet) + }, + writer: databricks_destination(), + expected_reason: "SQLLoad reads the files because at is a time", + }, + PlanRouteReasonTestCase { + description: "masked Parquet in S3 DuckDB could read itself names the mask", + reader: ReaderOffer { + needs_loader: Some("the table masks columns".to_owned()), + ..files_source("s3://lake/scores/", FileFormat::Parquet) + }, + writer: duckdb_destination(), + expected_reason: "SQLLoad reads the files because the table masks columns", + }, + PlanRouteReasonTestCase { + description: "masked Parquet into Postgres, which reads no files, says only that it can't read them", + reader: ReaderOffer { + needs_loader: Some("the table masks columns".to_owned()), + ..files_source("s3://lake/scores/", FileFormat::Parquet) + }, + writer: database_destination(), + expected_reason: "the destination cannot read the files where they are", + }, + ]; for test_case in test_cases { let reason = plan_route( &test_case.reader, diff --git a/crates/sqlload/src/store_paths/_helpers/locations.rs b/crates/sqlload/src/store_paths/_helpers/locations.rs index f7af877ec..75e46b737 100644 --- a/crates/sqlload/src/store_paths/_helpers/locations.rs +++ b/crates/sqlload/src/store_paths/_helpers/locations.rs @@ -27,6 +27,24 @@ pub(crate) fn url_location(path: &str) -> Result, PathError> { prefix: folder(&[key])?, })); } + if let Some(rest) = path.strip_prefix("gs://") { + let (bucket, key) = rest.split_once('/').unwrap_or((rest, "")); + if bucket.is_empty() { + return Err(PathError::Empty { + path: path.to_owned(), + store: "GCS account", + }); + } + return Ok(Some(Location { + store: StoreKind::Gcs { + bucket: bucket.to_owned(), + service_account_key: None, + endpoint: None, + hmac: None, + }, + prefix: folder(&[key])?, + })); + } let windows = cfg!(windows); let local = match path.strip_prefix("file://") { Some(rest) => windows_file_url_path(rest, windows), @@ -56,7 +74,8 @@ pub(crate) fn url_location(path: &str) -> Result, PathError> { pub(crate) fn same_store(connection: &StoreKind, url: &StoreKind) -> bool { match (connection, url) { - (StoreKind::S3 { bucket, .. }, StoreKind::S3 { bucket: other, .. }) => bucket == other, + (StoreKind::S3 { bucket, .. }, StoreKind::S3 { bucket: other, .. }) + | (StoreKind::Gcs { bucket, .. }, StoreKind::Gcs { bucket: other, .. }) => bucket == other, (StoreKind::Filesystem { .. }, StoreKind::Filesystem { .. }) => true, _ => false, } @@ -65,6 +84,7 @@ pub(crate) fn same_store(connection: &StoreKind, url: &StoreKind) -> bool { pub(crate) fn describe(store: &StoreKind) -> String { match store { StoreKind::S3 { bucket, .. } => format!("S3 bucket {bucket}"), + StoreKind::Gcs { bucket, .. } => format!("GCS bucket {bucket}"), StoreKind::Filesystem { root } => format!("filesystem {root}"), StoreKind::Sftp { server, .. } => format!("SFTP server {}", server.host), } diff --git a/crates/sqlload/src/store_paths/main/connection_store.rs b/crates/sqlload/src/store_paths/main/connection_store.rs new file mode 100644 index 000000000..0db40b4d2 --- /dev/null +++ b/crates/sqlload/src/store_paths/main/connection_store.rs @@ -0,0 +1,59 @@ +//! The store a connection's files are in. + +use crate::project::models::ConnectionConfig; +use crate::ssh_tunnel::models::SshTunnel; +use crate::store_paths::models::{HmacKey, StoreKind}; + +/// The store `connection` holds files in: an `s3` or `gcs` bucket, a `filesystem` root, or an `sftp` server below the login folder; `None` for a connection to anything else. +pub fn connection_store(connection: &ConnectionConfig) -> Option { + Some(match connection { + ConnectionConfig::S3 { + bucket, + region, + endpoint, + profile, + } => StoreKind::S3 { + bucket: bucket.clone(), + region: region.clone(), + endpoint: endpoint.clone(), + profile: profile.clone(), + }, + ConnectionConfig::Gcs { + bucket, + service_account_key, + endpoint, + hmac_key_id, + hmac_secret, + } => StoreKind::Gcs { + bucket: bucket.clone(), + service_account_key: service_account_key.clone(), + endpoint: endpoint.clone(), + hmac: hmac_key_id + .clone() + .zip(hmac_secret.clone()) + .map(|(key_id, secret)| HmacKey { key_id, secret }), + }, + ConnectionConfig::Filesystem { root } => StoreKind::Filesystem { root: root.clone() }, + ConnectionConfig::Sftp { + host, + port, + user, + private_key_file, + private_key_passphrase, + password, + known_hosts, + } => StoreKind::Sftp { + server: Box::new(SshTunnel { + host: host.clone(), + port: *port, + user: user.clone(), + private_key_file: private_key_file.clone(), + private_key_passphrase: private_key_passphrase.clone(), + password: password.clone(), + known_hosts: Some(known_hosts.clone()), + }), + root: String::new(), + }, + _ => return None, + }) +} diff --git a/crates/sqlload/src/store_paths/main/mod.rs b/crates/sqlload/src/store_paths/main/mod.rs index 544cd8210..c96b1f249 100644 --- a/crates/sqlload/src/store_paths/main/mod.rs +++ b/crates/sqlload/src/store_paths/main/mod.rs @@ -1,3 +1,4 @@ +pub mod connection_store; pub mod escape_partition_value; pub mod file_url_path; pub mod hive_values; diff --git a/crates/sqlload/src/store_paths/main/resolve_location.rs b/crates/sqlload/src/store_paths/main/resolve_location.rs index 506bc42c4..5b6bf962d 100644 --- a/crates/sqlload/src/store_paths/main/resolve_location.rs +++ b/crates/sqlload/src/store_paths/main/resolve_location.rs @@ -25,8 +25,10 @@ pub fn resolve_location( prefix: folder(&[path.strip_prefix("~/").unwrap_or(path)])?, }); } - let bucket_relative = - matches!(connection, Some((_, StoreKind::S3 { .. }))) && !path.contains("://"); + let bucket_relative = matches!( + connection, + Some((_, StoreKind::S3 { .. } | StoreKind::Gcs { .. })) + ) && !path.contains("://"); let from_url = match bucket_relative { true => None, false => url_location(path)?, diff --git a/crates/sqlload/src/store_paths/models.rs b/crates/sqlload/src/store_paths/models.rs index 9c335c5f8..fd329d8af 100644 --- a/crates/sqlload/src/store_paths/models.rs +++ b/crates/sqlload/src/store_paths/models.rs @@ -9,6 +9,15 @@ pub enum StoreKind { /// The shared credentials file's profile to take keys from, over `AWS_PROFILE`. profile: Option, }, + /// A Google Cloud Storage bucket, read with `service_account_key`, else Google's application default credentials. + Gcs { + bucket: String, + service_account_key: Option, + /// A custom endpoint (tests). + endpoint: Option, + /// The connection's HMAC key, with which DuckDB and MotherDuck read the bucket themselves. + hmac: Option, + }, /// A directory on the machine SQLLoad runs on. Filesystem { root: String }, /// A folder on an SFTP server: below `/` when `root` is `/`, otherwise below the user's login folder (`root` empty). @@ -29,6 +38,7 @@ impl std::fmt::Display for Location { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.store { StoreKind::S3 { bucket, .. } => write!(formatter, "s3://{bucket}/{}", self.prefix), + StoreKind::Gcs { bucket, .. } => write!(formatter, "gs://{bucket}/{}", self.prefix), StoreKind::Filesystem { root } => { write!(formatter, "{}/{}", root.trim_end_matches('/'), self.prefix) } @@ -43,3 +53,10 @@ impl std::fmt::Display for Location { } } } + +/// An HMAC key for a GCS bucket's S3-compatible API. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HmacKey { + pub key_id: String, + pub secret: crate::auth::models::Secret, +} diff --git a/crates/sqlload/src/store_paths/tests/helpers.rs b/crates/sqlload/src/store_paths/tests/helpers.rs index 7b94a39a1..0ef0eabda 100644 --- a/crates/sqlload/src/store_paths/tests/helpers.rs +++ b/crates/sqlload/src/store_paths/tests/helpers.rs @@ -10,6 +10,16 @@ pub(crate) fn bucket(name: &str) -> Option { }) } +/// A `gcs` connection's store for bucket `name`. +pub(crate) fn gcs_bucket(name: &str) -> Option { + Some(StoreKind::Gcs { + bucket: name.to_owned(), + service_account_key: None, + endpoint: None, + hmac: None, + }) +} + /// The root and folders of `path` as `local_root` finds them, written `root|folders`. pub(crate) fn root_of(path: &str, windows: bool) -> Option { let separated = crate::store_paths::_helpers::local_root::windows_separators(path, windows); diff --git a/crates/sqlload/src/store_paths/tests/test_resolve_location.rs b/crates/sqlload/src/store_paths/tests/test_resolve_location.rs index cfb219bfe..b5fb5d4bb 100644 --- a/crates/sqlload/src/store_paths/tests/test_resolve_location.rs +++ b/crates/sqlload/src/store_paths/tests/test_resolve_location.rs @@ -1,6 +1,6 @@ use crate::store_paths::main::resolve_location::resolve_location; use crate::store_paths::models::StoreKind; -use crate::store_paths::tests::helpers::bucket; +use crate::store_paths::tests::helpers::{bucket, gcs_bucket}; use crate::store_paths::tests::test_types::ResolveLocationTestCase; #[test] @@ -32,6 +32,38 @@ fn given_a_path_when_resolving_then_the_store_and_folder_are_found() { path: "s3://example-bucket-test/a//b", expected_location: Ok("s3://example-bucket-test/a/b/"), }, + ResolveLocationTestCase { + description: "a relative path is inside a gcs connection's bucket", + connection: gcs_bucket("example-bucket-prod"), + path: "events", + expected_location: Ok("gs://example-bucket-prod/events/"), + }, + ResolveLocationTestCase { + description: "a gs:// URL to a gcs connection's bucket is accepted", + connection: gcs_bucket("example-bucket-prod"), + path: "gs://example-bucket-prod/events/", + expected_location: Ok("gs://example-bucket-prod/events/"), + }, + ResolveLocationTestCase { + description: "a gs:// URL is not in an s3 connection's bucket of the same name", + connection: bucket("example-bucket-prod"), + path: "gs://example-bucket-prod/events/", + expected_location: Err( + "points at GCS bucket example-bucket-prod, but connection \"lake\" is S3 bucket example-bucket-prod", + ), + }, + ResolveLocationTestCase { + description: "without a connection gs:// picks GCS", + connection: None, + path: "gs://example-bucket-test/a//b", + expected_location: Ok("gs://example-bucket-test/a/b/"), + }, + ResolveLocationTestCase { + description: "a whole GCS bucket is rejected", + connection: None, + path: "gs:///x", + expected_location: Err("is empty"), + }, ResolveLocationTestCase { description: "an absolute path is the local filesystem", connection: None, diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index 8b642b76c..29dfa343f 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -63,6 +63,8 @@ mod test_files_round_trip; mod test_files_to_clickhouse; #[path = "loader/test_files_to_duckdb.rs"] mod test_files_to_duckdb; +#[path = "loader/test_gcs_files.rs"] +mod test_gcs_files; #[path = "loader/test_loader_edge_cases.rs"] mod test_loader_edge_cases; #[path = "loader/test_memory_budget.rs"] diff --git a/crates/sqlload/tests/loader/after_load/helpers.rs b/crates/sqlload/tests/loader/after_load/helpers.rs index 414171c1e..37781f3a4 100644 --- a/crates/sqlload/tests/loader/after_load/helpers.rs +++ b/crates/sqlload/tests/loader/after_load/helpers.rs @@ -167,7 +167,7 @@ fn zip_archive(members: &str) -> Result, String> { .map_err(|error| error.to_string()) } -/// Write `test_case`'s files into its folder of `bucket` at `endpoint`, list the source's `drop/` files, rewrite its `rewritten` ones, run its `after_load` on the listed files, and return the report as `action status files failed` and every file left in its folder as `key=contents`, sorted. +/// Write `test_case`'s files into its folder of the S3 `bucket` at `endpoint`, list the source's `drop/` files, rewrite its `rewritten` ones, run its `after_load` on the listed files, and return the report as `action status files failed` and every file left in its folder as `key=contents`, sorted. pub(crate) async fn run_s3_after_load( (endpoint, bucket): (&str, &str), test_case: &S3AfterLoadTestCase, @@ -178,6 +178,14 @@ pub(crate) async fn run_s3_after_load( endpoint: Some(endpoint.to_owned()), profile: None, }; + run_store_after_load(store, test_case).await +} + +/// [`run_s3_after_load`] in any bucket `store`. +pub(crate) async fn run_store_after_load( + store: StoreKind, + test_case: &S3AfterLoadTestCase, +) -> Result<(String, Vec), String> { let open = |prefix: String| { open_folder(&Location { store: store.clone(), diff --git a/crates/sqlload/tests/loader/files/helpers.rs b/crates/sqlload/tests/loader/files/helpers.rs index 61fd2e392..3573d286d 100644 --- a/crates/sqlload/tests/loader/files/helpers.rs +++ b/crates/sqlload/tests/loader/files/helpers.rs @@ -2,6 +2,7 @@ use arrow_array::RecordBatch; use sqlload::adapters::dispatch::main::session::execute::execute; use sqlload::adapters::dispatch::main::session::single_text::single_text; use sqlload::adapters::dispatch::models::Destination; +use sqlload::auth::models::Secret; use sqlload::file_store::main::list_files::list_files; use sqlload::file_store::main::open_folder::open_folder; use sqlload::file_store::main::read_file::read_file; @@ -11,7 +12,8 @@ use sqlload::loader::main::connect_targets::connect_targets; use sqlload::loader::main::plan_project::plan_project; use sqlload::loader::main::render_plan::render_plan; use sqlload::loader::models::PlanOptions; -use sqlload::store_paths::models::Location; +use sqlload::store_paths::models::{Location, StoreKind}; +use sqlload_testkit::fake_gcs::constants::{BUCKET, SERVICE_ACCOUNT_KEY}; use crate::helpers::{ column_values, day_files, described_project, duckdb_text, load_with_options, local_store, @@ -22,6 +24,20 @@ use crate::test_types::{ SourceRowUpgradeTestCase, }; +/// The fake GCS server's bucket as a `gcs` connection, and its `[connections.files]` TOML. +pub(crate) fn gcs_store(endpoint: &str, _: &std::path::Path) -> (StoreKind, String) { + let toml = format!( + "kind = \"gcs\"\nbucket = \"{BUCKET}\"\nendpoint = \"{endpoint}\"\nservice_account_key = '{SERVICE_ACCOUNT_KEY}'" + ); + let store = StoreKind::Gcs { + bucket: BUCKET.to_owned(), + service_account_key: Some(Secret(SERVICE_ACCOUNT_KEY.to_owned())), + endpoint: Some(endpoint.to_owned()), + hmac: None, + }; + (store, toml) +} + /// A run adding no source files. pub(crate) fn no_files() -> Result, String> { Ok(Vec::new()) diff --git a/crates/sqlload/tests/loader/test_files_to_duckdb.rs b/crates/sqlload/tests/loader/test_files_to_duckdb.rs index c5eb6f06c..0af93982a 100644 --- a/crates/sqlload/tests/loader/test_files_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_files_to_duckdb.rs @@ -76,7 +76,7 @@ const NOV_6_ESCAPED: ParquetFile = ParquetFile { extra: Some("w"), }; -const APPEND_RUNS: &[FilesRun] = &[ +pub(crate) const APPEND_RUNS: &[FilesRun] = &[ FilesRun { files: &[NOV_3, NOV_4], expected_error: "", @@ -88,7 +88,7 @@ const APPEND_RUNS: &[FilesRun] = &[ ]; /// Rows, distinct source files, and the newest `extra`; a column added in later files is NULL for older rows. -const APPEND_CHECK: &str = "SELECT count(*) || ',' || count(DISTINCT _sqlload_source_file) || ',' || count(extra) || ',' || max(day) FROM raw.scores"; +pub(crate) const APPEND_CHECK: &str = "SELECT count(*) || ',' || count(DISTINCT _sqlload_source_file) || ',' || count(extra) || ',' || max(day) FROM raw.scores"; /// `0` while the table was never created. const NO_TABLE: &str = @@ -1958,15 +1958,16 @@ fn given_an_append_parquet_table_with_move_to_when_reloading_then_each_archived_ score: 2.0, extra: None, }], - expected_plan: "route read parquet into Arrow -> DuckDB takes Arrow (--reload also reads the files after_load moved to", + expected_plan: "--reload also reads the files after_load moved to", expected_rows: "1:1.0,2:2.0", }]; for test_case in &test_cases { let (plan, rows) = run_archived_reload(test_case)?; assert_eq!(rows, test_case.expected_rows, "{}", test_case.description); - assert!( - plan.contains(test_case.expected_plan), - "{}:\n{plan}", + assert_eq!( + plan.matches(test_case.expected_plan).count(), + 1, + "{}, said once:\n{plan}", test_case.description ); } diff --git a/crates/sqlload/tests/loader/test_gcs_files.rs b/crates/sqlload/tests/loader/test_gcs_files.rs new file mode 100644 index 000000000..bd0ff6486 --- /dev/null +++ b/crates/sqlload/tests/loader/test_gcs_files.rs @@ -0,0 +1,190 @@ +use sqlload::files_source::models::AfterLoad; +use sqlload_testkit::fake_gcs::main::start_fake_gcs::start_fake_gcs; + +use crate::after_load_helpers::run_store_after_load; +use crate::files_helpers::{gcs_store, no_files, run_files_to_files, third_day}; +use crate::helpers::{plain, run_async, run_files_scenario, run_text_files_scenario, two_days}; +use crate::test_files_to_duckdb::{APPEND_CHECK, APPEND_RUNS}; +use crate::test_text_files_to_duckdb::{ + PATTERN_CHECK, PATTERN_RESULT, PATTERN_RUNS, PATTERN_SETTINGS, +}; +use crate::test_types::{ + FilesScenarioTestCase, FilesToFilesTestCase, S3AfterLoadTestCase, TextFile, TextFilesRun, + TextFilesScenarioTestCase, +}; + +/// The later of two dated files, by path, modified first, each holding P02. +const RESTATED: &[TextFile] = &[ + TextFile { + key: "sales_2026-11-14.csv", + contents: "le_version,line,amount\nP02,X,7\n", + encode: plain, + }, + TextFile { + key: "sales_2026-11-07.csv", + contents: "le_version,line,amount\nP01,A,1\nP02,A,10\n", + encode: plain, + }, +]; + +#[test] +fn given_parquet_files_in_gcs_when_appending_them_into_duckdb_then_they_load_as_from_s3() +-> Result<(), String> { + let test_cases = [FilesScenarioTestCase { + description: "Parquet appended from GCS loads each file once and adds columns from newer files", + store: gcs_store, + settings: "write_strategy = \"append\"\npartition_columns = { day = \"date\" }\n", + runs: APPEND_RUNS, + check_query: APPEND_CHECK, + expected_result: "5,4,3,2026-11-06", + }]; + run_async(|| async move { + let server = start_fake_gcs().await?; + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + for test_case in &test_cases { + let result = run_files_scenario(&server.endpoint, directory.path(), test_case).await?; + assert_eq!( + result.as_deref(), + Some(test_case.expected_result), + "{}", + test_case.description + ); + } + Ok(()) + }) +} + +#[test] +fn given_text_files_in_gcs_when_pattern_order_and_replace_key_pick_them_then_they_load_as_from_s3() +-> Result<(), String> { + let test_cases = [ + TextFilesScenarioTestCase { + description: "pattern picks pipe-delimited .txt files in GCS beside another, merged on their key in path order", + store: gcs_store, + settings: PATTERN_SETTINGS, + runs: PATTERN_RUNS, + check_query: PATTERN_CHECK, + expected_result: PATTERN_RESULT, + }, + TextFilesScenarioTestCase { + description: "replace_key with on_overlapping_files = \"last_wins\" in path order keeps P02 from the file last by path in GCS", + store: gcs_store, + settings: "format = \"csv\"\nwrite_strategy = \"delete_insert\"\nreplace_key = [\"le_version\"]\non_overlapping_files = \"last_wins\"\norder = \"path\"\n", + runs: &[TextFilesRun { + files: RESTATED, + expected_error: "", + expected_plan: "the last file holding a replace_key value replaces it", + }], + check_query: "SELECT string_agg(le_version || ':' || line || ':' || CAST(amount AS VARCHAR), ',' ORDER BY le_version, line) FROM raw.scores", + expected_result: "P01:A:1,P02:X:7", + }, + ]; + run_async(|| async move { + let server = start_fake_gcs().await?; + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + for test_case in &test_cases { + let result = + run_text_files_scenario(&server.endpoint, directory.path(), test_case).await?; + assert_eq!( + result.as_deref(), + Some(test_case.expected_result), + "{}", + test_case.description + ); + } + Ok(()) + }) +} + +#[test] +fn given_a_files_target_in_gcs_when_a_files_source_in_gcs_loads_into_it_then_each_strategy_holds_each_row_once() +-> Result<(), String> { + const APPEND: FilesToFilesTestCase = FilesToFilesTestCase { + description: "append writes each new day once", + store: gcs_store, + settings: "write_strategy = \"append\"", + runs: &[two_days, no_files, third_day], + expected_held: &["6|1", "6|1", "8|2"], + expected_reads: "", + }; + let test_cases = [ + FilesToFilesTestCase { ..APPEND }, + FilesToFilesTestCase { + description: "replace writes every row once, replacing the last load's files", + settings: "write_strategy = \"replace\"", + expected_held: &["6|1", "6|1", "8|1"], + ..APPEND + }, + FilesToFilesTestCase { + description: "delete_insert by day replaces the newest day's files, never doubling", + settings: "write_strategy = \"delete_insert\"\ncursor_column = \"day\"\ndestination_path = \"PREFIX/out/{table}/day={cursor:date}/\"", + expected_held: &["6|1", "6|2", "8|2"], + ..APPEND + }, + ]; + run_async(|| async move { + let server = start_fake_gcs().await?; + for test_case in &test_cases { + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + let (held, _) = + run_files_to_files(&server.endpoint, directory.path(), test_case).await?; + assert_eq!( + held, + test_case + .expected_held + .iter() + .map(|held| (*held).to_owned()) + .collect::>(), + "{}", + test_case.description + ); + } + Ok(()) + }) +} + +#[test] +fn given_files_in_gcs_when_after_load_acts_then_they_move_server_side_or_are_deleted_and_a_changed_file_stays() +-> Result<(), String> { + let test_cases = [ + S3AfterLoadTestCase { + description: "moves keep their key below move_to", + folder: "gcs-move/", + after_load: AfterLoad::MoveTo("gcs-move/done".to_owned()), + files: &[("drop/a.csv", "a"), ("drop/2026/b.csv", "b")], + rewritten: &[], + expected_report: "move done 2 0", + expected_files: &["done/2026/b.csv=b", "done/a.csv=a"], + }, + S3AfterLoadTestCase { + description: "deletes remove only the listed files, not one rewritten since", + folder: "gcs-delete/", + after_load: AfterLoad::Delete, + files: &[("drop/a.csv", "a"), ("drop/b.csv", "b")], + rewritten: &[("drop/a.csv", "a, longer now")], + expected_report: "delete failed 1 1", + expected_files: &["drop/a.csv=a, longer now"], + }, + ]; + run_async(|| async move { + let server = start_fake_gcs().await?; + for test_case in &test_cases { + let (store, _) = gcs_store(&server.endpoint, std::path::Path::new("")); + let actual = run_store_after_load(store, test_case).await?; + assert_eq!( + actual, + ( + test_case.expected_report.to_owned(), + test_case + .expected_files + .iter() + .map(|file| (*file).to_owned()) + .collect() + ), + "{}", + test_case.description + ); + } + Ok(()) + }) +} diff --git a/crates/sqlload/tests/loader/test_text_files_to_duckdb.rs b/crates/sqlload/tests/loader/test_text_files_to_duckdb.rs index 241ad7ef2..a9443737d 100644 --- a/crates/sqlload/tests/loader/test_text_files_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_text_files_to_duckdb.rs @@ -99,18 +99,18 @@ const PATTERN_FILES: &[TextFile] = &[ }, ]; -const PATTERN_SETTINGS: &str = "format = \"csv\"\ncsv = { delimiter = \"|\" }\npattern = \"file_*.txt\"\nwrite_strategy = \"merge\"\nunique_key = [\"id\"]\norder = \"path\"\n"; +pub(crate) const PATTERN_SETTINGS: &str = "format = \"csv\"\ncsv = { delimiter = \"|\" }\npattern = \"file_*.txt\"\nwrite_strategy = \"merge\"\nunique_key = [\"id\"]\norder = \"path\"\n"; -const PATTERN_RUNS: &[TextFilesRun] = &[TextFilesRun { +pub(crate) const PATTERN_RUNS: &[TextFilesRun] = &[TextFilesRun { files: PATTERN_FILES, expected_error: "", expected_plan: "2 new of 2 matching file_*.txt, 1 other file ignored", }]; /// Each row's key, value and the name of the file it came from. -const PATTERN_CHECK: &str = "SELECT string_agg(id || v || '@' || regexp_extract(_sqlload_source_file, '[^/]+$'), ',' ORDER BY id) FROM raw.scores"; +pub(crate) const PATTERN_CHECK: &str = "SELECT string_agg(id || v || '@' || regexp_extract(_sqlload_source_file, '[^/]+$'), ',' ORDER BY id) FROM raw.scores"; -const PATTERN_RESULT: &str = "1a@file_1.txt,2B@file_2.txt,3c@file_2.txt"; +pub(crate) const PATTERN_RESULT: &str = "1a@file_1.txt,2B@file_2.txt,3c@file_2.txt"; /// `code,amount` rows: 10,000 sampled ones with numeric codes, then an alphanumeric code and one with leading zeros. const CODES_AFTER_SAMPLE: TextFile = TextFile { diff --git a/fensu.toml b/fensu.toml index 4ab2b70bd..1bff2c0dc 100644 --- a/fensu.toml +++ b/fensu.toml @@ -277,6 +277,11 @@ rule = "FPRSL105" path = "crates/sqlload-testkit/src/seaweedfs/main/start_seaweedfs.rs" reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlload-testkit/src/fake_gcs/main/start_fake_gcs.rs" +reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." + [[targets.rust.rule_exceptions]] rule = "FPRSL105" path = "crates/sqlload/src/postgres_source/main/decode_copy.rs" diff --git a/website/public/index.html b/website/public/index.html index 084173875..3a2778a52 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -173,7 +173,7 @@

See every change first

02 · Load

Bulk copy, not row inserts

-

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

+

Files in S3 load into Snowflake straight from an external stage, and Parquet in S3 into Databricks through an external location. Parquet loads into DuckDB through DuckDB's own reader, and Parquet in S3 or Google Cloud Storage into MotherDuck on its servers. Snowflake tables load into Snowflake on the same connection, or through an external stage that covers the unload, without leaving Snowflake. Snowflake tables unloaded to S3 load into DuckDB, Databricks and ClickHouse with no row passing through SQLLoad. Rows are staged as Parquet and copied in bulk.

Load methods and routes →

sqlload plan
@@ -284,7 +284,7 @@ 

Start in a minute

DuckDB✓ MotherDuck Beta✓ ODBC✓ SQL Server, MySQL, OpenEdge - Files✓ Parquet, CSV, JSON, Excel, XML✓ Parquet, CSV + Files: S3, GCS, local, SFTP✓ Parquet, CSV, JSON, Excel, XML✓ Parquet, CSV Google Sheets✓ APIs, Python✓ Kafka✓ to Snowflake, Databricks, BigQuery diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index 433349e72..9d5bb98f1 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -146,6 +146,33 @@ keys. SQLLoad removes files in bulk, up to 1,000 per request. For an S3-compatible store without bulk delete, set `AWS_DISABLE_BULK_DELETE=true`: SQLLoad then deletes each file with its own request. +### `gcs` + +A Google Cloud Storage bucket, for [files sources](/docs/sources/files/) and files targets. + +| Key | Meaning | +|-----|---------| +| `bucket` | The bucket. | +| `service_account_key` | A service account's JSON key, such as `"${ENV:GCS_KEY}"`. | +| `hmac_key_id`, `hmac_secret` | An HMAC key for the bucket, which [DuckDB](/docs/targets/duckdb/#parquet-files-in-gcs) and [MotherDuck](/docs/targets/motherduck/#parquet-files-in-gcs) need to read its Parquet files themselves. Set both or neither; write the secret as `"${ENV:NAME}"`. | +| `endpoint` | A custom endpoint, such as a GCS emulator's. | + +SQLLoad reads and writes the bucket with the first credentials it finds, in this order: + +1. The connection's `service_account_key`. +2. The key file `GOOGLE_APPLICATION_CREDENTIALS` names, which must exist. +3. gcloud's application default credentials (`gcloud auth application-default login`) (Beta). +4. The service account of the machine SQLLoad runs on, from its metadata server (Beta). + +They need `roles/storage.objectViewer` on the bucket to read it, and `roles/storage.objectUser` +to write files or remove them after a load. `sqlload debug` shows where each GCS connection's +credentials come from, never the credentials. + +DuckDB and MotherDuck read GCS only with an HMAC key, not a service-account key. To have them read +a bucket's Parquet files themselves, create an HMAC key for a service account that can read it +(Cloud Storage → Settings → Interoperability) and set `hmac_key_id` and `hmac_secret`. Without one, +SQLLoad reads the files and sends their rows, and `sqlload plan -v ` says why. + ### `filesystem` | Key | Meaning | diff --git a/website/src/content/docs/docs/guides/files.mdx b/website/src/content/docs/docs/guides/files.mdx index dc2e2ea2d..cdc27a594 100644 --- a/website/src/content/docs/docs/guides/files.mdx +++ b/website/src/content/docs/docs/guides/files.mdx @@ -1,10 +1,10 @@ --- title: "Load files" -description: "Load Parquet, CSV, JSON or NDJSON files from a local folder, S3 or an SFTP server, load only new files or new partitions, and let the warehouse read the files itself." +description: "Load Parquet, CSV, JSON or NDJSON files from a local folder, S3, Google Cloud Storage or an SFTP server, load only new files or new partitions, and let the warehouse read the files itself." --- A files source loads every file under a folder into one table. This guide loads a local folder, -then S3 and an SFTP server, and shows when the warehouse reads the files itself. Every setting is +then S3, Google Cloud Storage and an SFTP server, and shows when the warehouse reads the files itself. Every setting is on the [Files source](/docs/sources/files/) page. ## Load a folder @@ -129,6 +129,23 @@ SQLLoad finds AWS credentials in the environment, the shared credentials file, w the instance or container role. See the [`s3` connection](/docs/configuration/#s3) for the order, `profile` and custom endpoints. +## Load files from Google Cloud Storage + +A `gcs` connection names a bucket, and the source is the same as for S3: + +```toml +[connections.lake] +kind = "gcs" +bucket = "example-bucket" +service_account_key = "${ENV:GCS_KEY}" +hmac_key_id = "GOOG1EXAMPLE" +hmac_secret = "${ENV:GCS_HMAC_SECRET}" +``` + +Without `service_account_key`, SQLLoad uses Google's application default credentials. The HMAC +key is optional: DuckDB and MotherDuck read Parquet in GCS themselves only with one. See the +[`gcs` connection](/docs/configuration/#gcs). + ## Load files from an SFTP server An `sftp` connection reads files from an SFTP server, such as a partner's daily drop: @@ -169,7 +186,7 @@ SQLLoad. `plan` names the route on each table's `route` line: | Destination | Reads the files itself | |-------------|------------------------| -| DuckDB | Parquet in S3 or a local folder, with `read_parquet`. | +| DuckDB | Parquet in S3, GCS (with an HMAC key) or a local folder, with `read_parquet`. | | Snowflake | Parquet, JSON, NDJSON and CSV under the `url` of an `external_stage` load method, with `COPY`. | | ClickHouse | Parquet in S3, with `s3()`. | diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index 78bc68f51..a4661a133 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -3,8 +3,8 @@ title: "Files" description: "The files source: settings, which files load, columns, columns from paths, whole records, SFTP servers and the files a destination reads itself." --- -A files source is one table built from every file under a folder, in S3, a local folder or an -SFTP server. To set one up, see [Load files](/docs/guides/files/). +A files source is one table built from every file under a folder, in S3, Google Cloud Storage, a +local folder or an SFTP server. To set one up, see [Load files](/docs/guides/files/). ```toml [connections.lake] @@ -25,7 +25,7 @@ cursor_column = "dt" | Setting | Meaning | |---------|---------| -| `connection` | An `s3` or `filesystem` connection, or an `sftp` connection. | +| `connection` | An `s3`, `gcs` or `filesystem` connection, or an `sftp` connection. | | `path` | The folder holding the files, relative to the connection. | | `format` | `parquet` (default), `ndjson` (one object per line), `json` (objects one after another, or one top-level array) or `csv`; also `xlsx` and `xml`. See [Excel, XML and ZIP files](#excel-xml-and-zip-files). | | `compression` | For text formats: `auto` (default, by file name: `.gz` is gzip, `.zst` is zstd), `none`, `gzip` or `zstd`; also `zip`, which reads the `format`'s files inside `.zip` archives. Parquet compresses inside the file. | @@ -667,7 +667,8 @@ after_load = { move_to = "loaded" } `loaded/2026/a.csv`. - A file already at the destination is never replaced: both stay. On a local disk a move is a hard link, then a delete; where a hard link fails, as across volumes, the file is copied into a new - file that is created only if none is there. S3 has no copy that refuses to replace a file, so + file that is created only if none is there. GCS copies only if no file is there. + S3 has no copy that refuses to replace a file, so SQLLoad checks the destination is free, then copies on the server: a file written to that same key at that moment would be replaced. S3 copies files of up to 5 GB; a larger one stays where it is and is reported. Either way the copy's size is checked before the source is deleted. @@ -726,7 +727,7 @@ on_empty = "skip" - Only an existing folder holding no file at all (hidden ones aside) counts as empty. A missing folder, one that can't be read, and one holding files of another format or that don't match `path_columns` still fail. -- S3 and SFTP list a folder that doesn't exist as empty, so SQLLoad can't tell a mistyped `path` +- S3, GCS and SFTP list a folder that doesn't exist as empty, so SQLLoad can't tell a mistyped `path` from an empty one there. On those connections `on_empty = "skip"` skips an empty folder only once its table exists; before its first load it fails, naming the path. - `after_load` already skips a folder it emptied once its table exists, without `on_empty`. @@ -877,7 +878,9 @@ set one up, see [Load files from an SFTP server](/docs/guides/files/#load-files- When the destination can load the files exactly by itself, it does, and no row passes through SQLLoad: -- **DuckDB** reads Parquet in S3 or a local folder with `read_parquet`. +- **DuckDB** reads Parquet in S3 or a local folder with `read_parquet`, and in GCS when the `gcs` + connection has an HMAC key. +- **MotherDuck** reads Parquet in S3, and in GCS with an HMAC key, on its servers. - **Snowflake** runs `COPY` from an external stage over the files' bucket, for Parquet, JSON, NDJSON and CSV. - **ClickHouse** reads Parquet in S3 with `s3()`. diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index 9fd512d30..ae2860dd2 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -16,6 +16,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | DuckDB | Tables and views of a database file, by `include` patterns, and query results | One `SELECT` on a read-only connection, streamed as Arrow. See [DuckDB](/docs/sources/duckdb/). | | MotherDuck (beta) | Tables and views of one database, by `include` patterns, and query results | One `SELECT` run by a separate DuckDB 1.5 process (`sqlload[motherduck]`), its Arrow batches handed over one at a time. See [MotherDuck](/docs/sources/motherduck/). | | Files in S3 | Parquet, CSV, JSON, NDJSON, Excel (xlsx) and XML; gzip or zstd for text formats; ZIP archives of any of them; any file name through `pattern`. See [Files](/docs/sources/files/#excel-xml-and-zip-files). | The destination itself where it can load them exactly, otherwise SQLLoad's reader | +| Files in Google Cloud Storage | As for S3 | As for S3; DuckDB and MotherDuck read Parquet themselves with an HMAC key. See [`gcs`](/docs/configuration/#gcs). | | Files in a local folder | As for S3 | As for S3 | | Files on an SFTP server | As for S3 | SQLLoad's reader, over SFTP. See [Files](/docs/sources/files/#files-on-an-sftp-server). | | Google Sheets | Tabs or A1 ranges, values typed by their cells | The Sheets API with a service-account key, into Arrow. See [Google Sheets](/docs/sources/google-sheets/). | @@ -33,8 +34,9 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | ClickHouse, one self-hosted server | `append`, `merge`, `delete_insert`, `replace` | Arrow over HTTP into a staging table, published by swapping in a whole table or whole partitions; Parquet in S3 read by ClickHouse with `s3()`. No clusters, replicated or distributed tables; ClickHouse Cloud is untested. See [ClickHouse](/docs/targets/clickhouse/). | | Postgres | `append`, `merge`, `delete_insert`, `replace` | Binary `COPY` into a temporary staging table, then one transaction per commit. See [Postgres](/docs/targets/postgres/). | | DuckDB | `append`, `merge`, `delete_insert`, `replace` | Arrow appends; Parquet files read by DuckDB with `read_parquet`. | -| MotherDuck (beta) | `append`, `merge`, `delete_insert`, `replace` | Batches inserted by a separate DuckDB 1.5 process (`sqlload[motherduck]`), as MotherDuck doesn't accept DuckDB 2.0 clients yet; Parquet in S3 is read on MotherDuck's servers, other files by SQLLoad. See [MotherDuck](/docs/targets/motherduck/). | +| MotherDuck (beta) | `append`, `merge`, `delete_insert`, `replace` | Batches inserted by a separate DuckDB 1.5 process (`sqlload[motherduck]`), as MotherDuck doesn't accept DuckDB 2.0 clients yet; Parquet in S3, and in GCS with an HMAC key, is read on MotherDuck's servers, other files by SQLLoad. See [MotherDuck](/docs/targets/motherduck/). | | Parquet or CSV files in S3 | `append`, `delete_insert`, `replace` | Parquet written by SQLLoad, zstd-compressed; or CSV: plain, gzip or zstd. | +| Parquet or CSV files in Google Cloud Storage | As for S3 | As for S3 | | Parquet or CSV files in a local folder | As for S3; `replace` swaps the folder in one rename | As for S3 | ## Source and destination combinations diff --git a/website/src/content/docs/docs/targets/duckdb.mdx b/website/src/content/docs/docs/targets/duckdb.mdx index 01120a09d..abc0c451b 100644 --- a/website/src/content/docs/docs/targets/duckdb.mdx +++ b/website/src/content/docs/docs/targets/duckdb.mdx @@ -6,7 +6,7 @@ description: "Load into DuckDB: Arrow appends, and Parquet files DuckDB reads it Rows from Postgres and Python sources are appended to DuckDB as Arrow, inside the batch's transaction. -Parquet files in S3 or a local folder load directly: DuckDB reads them with `read_parquet`, with the +Parquet files in S3, GCS or a local folder load directly: DuckDB reads them with `read_parquet`, with the same values SQLLoad's own reader gives: ```console @@ -27,6 +27,21 @@ SQLLoad's reader loads a files table instead when DuckDB can't load it exactly: A later batch whose files hold a column differently fails the load, naming the column. +### Parquet files in GCS + +Parquet files in a [`gcs` connection](/docs/configuration/#gcs) load the same way when the +connection sets an HMAC key (`hmac_key_id` and `hmac_secret`): DuckDB reads `gs://` with it, as a +secret that lasts only for the load's session. DuckDB takes no service-account key, so without an +HMAC key, or with a custom `endpoint`, SQLLoad reads the files instead, and `sqlload plan -v
` says why. `plan` +reads the first file to load with DuckDB, and fails the table with the fix when it can't, since the +HMAC key isn't what SQLLoad's own reader uses: + +```console +$ sqlload plan -v +… + ├── route DuckDB reads directly from gs://example-bucket/events/ (nothing through the loader) +``` + DuckDB reads S3 with the keys SQLLoad finds for the [`s3` connection](/docs/configuration/#s3): from the environment or the shared credentials file. Without either, it uses the AWS credential chain, which it resolves again every four minutes as its keys may diff --git a/website/src/content/docs/docs/targets/files.mdx b/website/src/content/docs/docs/targets/files.mdx index 22cf361a7..7ab0ad2ca 100644 --- a/website/src/content/docs/docs/targets/files.mdx +++ b/website/src/content/docs/docs/targets/files.mdx @@ -1,6 +1,6 @@ --- title: "Parquet and CSV files" -description: "Write each table as zstd Parquet or CSV files in S3 or a local folder, partitioned by a template." +description: "Write each table as zstd Parquet or CSV files in S3, Google Cloud Storage or a local folder, partitioned by a template." --- A files target writes each table as Parquet or CSV files in its own folder. @@ -17,7 +17,7 @@ path = "{source}/{table}/dt={cursor:date}/" | Key | Meaning | |-----|---------| -| `connection` | An `s3` or `filesystem` connection. | +| `connection` | An `s3`, `gcs` or `filesystem` connection. | | `path` | Each table's folder: a template with `{source}`, `{schema}`, `{table}`, and `{cursor}` or `{cursor:date}` to partition by the cursor. Default `{source}/{table}/`. | | `format` | `parquet` (default), or `csv`. See [CSV files](#csv-files). | | `compression` | CSV only: `none` (default), `gzip` or `zstd`. | diff --git a/website/src/content/docs/docs/targets/motherduck.mdx b/website/src/content/docs/docs/targets/motherduck.mdx index 9181f6afc..e38766705 100644 --- a/website/src/content/docs/docs/targets/motherduck.mdx +++ b/website/src/content/docs/docs/targets/motherduck.mdx @@ -93,6 +93,15 @@ $ sqlload plan └── route DuckDB reads directly from s3://example-bucket/events/ (nothing through the loader) ``` +### Parquet files in GCS + +Parquet files in a [`gcs` connection](/docs/configuration/#gcs) with an HMAC key (`hmac_key_id` +and `hmac_secret`) load the same way: MotherDuck's servers read `gs://` with a session-only secret +made from that key, and `plan` reads the first file on MotherDuck's servers first. Without an HMAC +key SQLLoad reads the files, as MotherDuck takes no service-account key. The key's service account +needs `roles/storage.objectViewer` on the bucket, and MotherDuck's servers read it from their own +network, so a VPC Service Controls perimeter around the bucket keeps them out. + The route loads the same values as SQLLoad's reader, and goes through SQLLoad's reader for the same reasons as on [DuckDB](/docs/targets/duckdb/), such as CSV or JSON files, masked columns or a custom S3 endpoint, which MotherDuck's servers don't reach. Each batch fails unless MotherDuck