diff --git a/crates/sqlload/src/http_source/_helpers/endpoint.rs b/crates/sqlload/src/http_source/_helpers/endpoint.rs index d04d13396..780f5251b 100644 --- a/crates/sqlload/src/http_source/_helpers/endpoint.rs +++ b/crates/sqlload/src/http_source/_helpers/endpoint.rs @@ -1,4 +1,4 @@ -//! A connection's settings resolved into what requests need, and the rate limit its jobs share. +//! A connection's settings resolved into what requests need, and the limits every request to it shares. use crate::load_events::main::emit::emit; use crate::load_events::models::EventKind; @@ -12,52 +12,70 @@ use serde_json::{Map, Value}; use crate::http_source::_helpers::oauth::shared_token_source; use crate::http_source::_helpers::urls::encoded; use crate::http_source::constants::SECRET_HEADER_WORDS; -use crate::http_source::models::{Auth, ClientSpec, Credential, Endpoint, Grant, Limiter}; +use crate::http_source::models::{ + Auth, ClientSpec, ConnectionLimits, Credential, Endpoint, Grant, HttpSettings, Limiter, +}; + +/// The settings of `client`'s connection that requests share, and the connection's own settings. +pub(crate) fn connection_settings<'a>( + client: &ClientSpec, + connections: &'a Map, +) -> Result<(HttpSettings, &'a Map), String> { + let name = &client.connection; + let settings = connections + .get(name) + .and_then(Value::as_object) + .ok_or_else(|| { + format!("sl.http uses connection {name}, which is not in sqlload.toml or is a target's connection") + })?; + let shared = serde_json::from_value::(Value::Object(settings.clone())) + .map_err(|error| format!("[connections.{name}]: {error}"))?; + Ok((shared, settings)) +} /// The endpoint of `client`'s connection: its `base_url`, headers and the credential its `auth` names. pub(crate) fn endpoint( client: &ClientSpec, - connections: &Map, + (auth, settings): (Option<&Auth>, &Map), ) -> Result { let name = &client.connection; - let settings = connections.get(name).ok_or_else(|| { - format!("sl.http uses connection {name}, which is not in sqlload.toml or is a target's connection") - })?; let text = |key: &str| settings.get(key).and_then(Value::as_str).map(str::to_owned); let base = text("base_url").ok_or_else(|| format!("connection {name} needs base_url for sl.http"))?; let base_url = reqwest::Url::parse(&base) .map_err(|error| format!("connection {name}: base_url {base} is not a URL: {error}"))?; let needed = |key: &str| { - text(key).filter(|value| !value.is_empty()).ok_or_else(|| { - format!("connection {name} needs {key} for the sl.http auth this job uses") - }) + text(key) + .filter(|value| !value.is_empty()) + .ok_or_else(|| format!("connection {name} needs {key} for its auth")) }; let present = |key: &str| text(key).filter(|value| !value.is_empty()); - let credential = match &client.auth { - Auth::None => Credential::None, - Auth::Bearer => Credential::Header { + let credential = match auth { + None => Credential::None, + Some(Auth::Bearer {}) => Credential::Header { name: "Authorization".to_owned(), value: format!("Bearer {}", needed("token")?), }, - Auth::ApiKey { + Some(Auth::ApiKey { header: Some(header), query: None, - } => Credential::Header { + }) => Credential::Header { name: header.clone(), value: needed("token")?, }, - Auth::ApiKey { + Some(Auth::ApiKey { header: None, query: Some(query), - } => Credential::Query { + }) => Credential::Query { name: query.clone(), value: needed("token")?, }, - Auth::ApiKey { .. } => { - return Err("sl.api_key takes exactly one of header= or query=".to_owned()); + Some(Auth::ApiKey { .. }) => { + return Err(format!( + "[connections.{name}] auth api_key takes exactly one of header or query" + )); } - Auth::Basic => { + Some(Auth::Basic {}) => { let pair = format!("{}:{}", needed("username")?, needed("password")?); Credential::Header { name: "Authorization".to_owned(), @@ -67,31 +85,31 @@ pub(crate) fn endpoint( ), } } - Auth::Oauth2ClientCredentials(spec) => { + Some(Auth::Oauth2ClientCredentials(spec)) => { let grant = Grant::ClientCredentials { client_id: needed("client_id")?, client_secret: needed("client_secret")?, }; Credential::OAuth2(shared_token_source( name, - &format!("{:?}", client.auth), + &format!("{auth:?}"), spec, grant, )?) } - Auth::Oauth2RefreshToken(spec) => { + Some(Auth::Oauth2RefreshToken(spec)) => { let grant = Grant::RefreshToken { refresh_token: needed("refresh_token")?, client: (present("client_id"), present("client_secret")), }; Credential::OAuth2(shared_token_source( name, - &format!("{:?}", client.auth), + &format!("{auth:?}"), spec, grant, )?) } - Auth::Oauth2Password(spec) => { + Some(Auth::Oauth2Password(spec)) => { let grant = Grant::Password { username: needed("username")?, password: needed("password")?, @@ -99,7 +117,7 @@ pub(crate) fn endpoint( }; Credential::OAuth2(shared_token_source( name, - &format!("{:?}", client.auth), + &format!("{auth:?}"), spec, grant, )?) @@ -179,27 +197,42 @@ fn text_headers(headers: &Map) -> BTreeMap { .collect() } -/// The limiter every job of `connection` with the limit `rate` (`10/s`, `600/m`, `1000/h`) shares in this process. -pub(crate) fn shared_limiter(connection: &str, rate: &str) -> Result, String> { - static LIMITERS: OnceLock>>> = OnceLock::new(); - let interval = interval(rate)?; - let limiters = LIMITERS.get_or_init(Mutex::default); - let mut limiters = limiters +/// The limiter and in-flight cap every request to `connection` shares in this process, made for its `limits` by the first job to call it. A later project in the same process (as tests run) that gives the connection other limits replaces them. +pub(crate) fn shared_limits( + connection: &str, + limits: &ConnectionLimits, +) -> Result { + static SHARED: OnceLock>> = + OnceLock::new(); + let mut shared = SHARED + .get_or_init(Mutex::default) .lock() .map_err(|_| "the rate limiters' lock is poisoned".to_owned())?; - let limiter = limiters - .entry(format!("{connection}\u{0}{rate}")) - .or_insert_with(|| { - Arc::new(Limiter { - connection: connection.to_owned(), - rate: rate.to_owned(), - interval, - next: Mutex::new(tokio::time::Instant::now()), - }) - }); - Ok(limiter.clone()) + if let Some((made_for, made)) = shared.get(connection) + && made_for == limits + { + return Ok(made.clone()); + } + let limiter = match &limits.rate_limit { + Some(rate) => Some(Arc::new(Limiter { + connection: connection.to_owned(), + rate: rate.clone(), + interval: interval(rate)?, + next: Mutex::new(tokio::time::Instant::now()), + })), + None => None, + }; + let in_flight = limits + .max_in_flight + .map(|most| Arc::new(tokio::sync::Semaphore::new(most))); + let made = (limiter, in_flight); + shared.insert(connection.to_owned(), (limits.clone(), made.clone())); + Ok(made) } +/// A connection's shared limiter and in-flight cap, each `None` when it sets no such limit. +pub(crate) type SharedLimits = (Option>, Option>); + /// The time between requests `rate` allows. pub(crate) fn interval(rate: &str) -> Result { let invalid = diff --git a/crates/sqlload/src/http_source/_helpers/fetch.rs b/crates/sqlload/src/http_source/_helpers/fetch.rs index a6f4dc95b..ea39fc46d 100644 --- a/crates/sqlload/src/http_source/_helpers/fetch.rs +++ b/crates/sqlload/src/http_source/_helpers/fetch.rs @@ -1,4 +1,4 @@ -//! One request with the shared rate limit, retries with backoff, `Retry-After`, and `on_status`. +//! One request under the connection's shared limits, retries with backoff, `Retry-After`, and `on_status`. use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -14,7 +14,7 @@ use crate::http_source::constants::{ use crate::http_source::models::{Attempt, Calls, Credential, Fetched, PageCall, StatusAction}; use crate::python_sources::errors::ReadError; -/// The response to `call`, its JSON body resent on every retry, or `None` when `on_status` skips its status; transport errors, 429 and 5xx are retried up to `calls.retries` times, and are still worth retrying later once those run out. +/// The response to `call`, its JSON body resent on every retry, or `None` when `on_status` skips its status; transport errors, 429 and 5xx are retried up to `calls.retries` times, and are still worth retrying later once those run out. Every attempt, retries included, holds one of the connection's `max_in_flight` permits while its request is open and takes a turn under its `rate_limit`. The permit comes first, then the OAuth2 access token, then the turn, so a request that got its turn starts at once: neither a queue for a permit nor a token fetch lets turns pass unused and then go in a burst. The token is taken again after the turn, which returns the one just fetched unless the wait outlasted it. The permit is released once the response is read, not held through a retry's backoff. pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result, ReadError> { let policy = RetryPolicy { first_delay: FIRST_RETRY_DELAY, @@ -25,7 +25,23 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result = None; let mut renewed = false; loop { - let waited = async { + let acquired = async { + match &calls.in_flight { + Some(in_flight) => Some(in_flight.clone().acquire_owned().await), + None => None, + } + }; + let permit = tokio::select! { + biased; + () = calls.stop.cancelled() => return Err(STOPPED.to_owned().into()), + permit = acquired => permit, + }; + if let Credential::OAuth2(tokens) = &calls.endpoint.credential { + tokens + .access_token(calls.retries, refused.as_deref()) + .await?; + } + let turn = async { if let Some(limiter) = &calls.limiter { limiter.wait().await; } @@ -33,7 +49,7 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result return Err(STOPPED.to_owned().into()), - () = waited => {} + () = turn => {} } let access = match &calls.endpoint.credential { Credential::OAuth2(tokens) => Some( @@ -46,7 +62,9 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result return Ok(fetched), Attempt::Refused => { refused = access; diff --git a/crates/sqlload/src/http_source/_helpers/job.rs b/crates/sqlload/src/http_source/_helpers/job.rs index fb8721ddc..2dfbef0a9 100644 --- a/crates/sqlload/src/http_source/_helpers/job.rs +++ b/crates/sqlload/src/http_source/_helpers/job.rs @@ -11,8 +11,7 @@ use tokio::sync::mpsc::{Receiver, Sender, channel}; use tokio::task::JoinSet; use crate::http_source::_helpers::cells::cells_problem; -use crate::http_source::_helpers::endpoint::{endpoint, interval, shared_limiter}; -use crate::http_source::_helpers::oauth::token_url; +use crate::http_source::_helpers::endpoint::{connection_settings, endpoint, shared_limits}; use crate::http_source::_helpers::pages::{paginate, set_at}; use crate::http_source::_helpers::reshaped::reshapes; use crate::http_source::_helpers::selector::{enclosing_levels, selected_rows, selector}; @@ -28,10 +27,11 @@ use std::time::Duration; use crate::http::main::install_tls_provider::install_tls_provider; use crate::http_source::constants::{ - BODY_DROPPING_REDIRECTS, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD, SHOWN_KEYS, + BODY_DROPPING_REDIRECTS, DEFAULT_RETRIES, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD, + SHOWN_KEYS, }; use crate::http_source::models::{ - Auth, Calls, FanKey, FanOut, FanOutLevel, HttpJob, HttpPage, KeyFeed, Location, Method, Over, + Calls, FanKey, FanOut, FanOutLevel, HttpJob, HttpPage, KeyFeed, Location, Method, Over, Paginator, Request, RowContext, Selector, Sink, StatusAction, }; @@ -188,12 +188,6 @@ fn request_problems(request: &Request) -> Vec { problems.extend(body_problems(request)); problems.extend(request.cells.as_ref().and_then(cells_problem)); problems.extend(request.error_path.as_deref().and_then(error_path_problem)); - problems.extend(match &request.client.auth { - Auth::Oauth2ClientCredentials(spec) - | Auth::Oauth2RefreshToken(spec) - | Auth::Oauth2Password(spec) => token_url(spec).err(), - Auth::None | Auth::Bearer | Auth::ApiKey { .. } | Auth::Basic => None, - }); problems.extend( request .client @@ -201,13 +195,6 @@ fn request_problems(request: &Request) -> Vec { .iter() .filter_map(|(name, value)| header_problem(name, value)), ); - problems.extend( - request - .client - .rate_limit - .as_deref() - .and_then(|rate| interval(rate).err()), - ); problems.extend( request .on_status @@ -708,11 +695,8 @@ fn request_calls( (http, source_part): (&reqwest::Client, Option), ) -> Result { let client = &request.client; - let limiter = client - .rate_limit - .as_deref() - .map(|rate| shared_limiter(&client.connection, rate)) - .transpose()?; + let (shared, settings) = connection_settings(client, connections)?; + let (limiter, in_flight) = shared_limits(&client.connection, &shared.limits)?; let on_status = request .on_status .iter() @@ -725,9 +709,10 @@ fn request_calls( .collect::, String>>()?; Ok(Calls { http: http.clone(), - endpoint: endpoint(client, connections)?, + endpoint: endpoint(client, (shared.auth.as_ref(), settings))?, limiter, - retries: client.retries, + in_flight, + retries: shared.retries.unwrap_or(DEFAULT_RETRIES), on_status, full_page_seen: std::sync::Arc::default(), source_part, diff --git a/crates/sqlload/src/http_source/constants.rs b/crates/sqlload/src/http_source/constants.rs index ad58c7b0d..a2707b00f 100644 --- a/crates/sqlload/src/http_source/constants.rs +++ b/crates/sqlload/src/http_source/constants.rs @@ -1,5 +1,14 @@ use std::time::Duration; +/// Retries of a failed request when the connection sets no `retries`. +pub const DEFAULT_RETRIES: u32 = 5; + +/// The most retries a connection may set. +pub const MAX_RETRIES: u32 = 100; + +/// The largest `max_in_flight` a connection may set. +pub const MAX_IN_FLIGHT: usize = 10_000; + /// The delay before the first retry; each later one doubles it. pub const FIRST_RETRY_DELAY: Duration = Duration::from_millis(500); diff --git a/crates/sqlload/src/http_source/main/http_connection_line.rs b/crates/sqlload/src/http_source/main/http_connection_line.rs new file mode 100644 index 000000000..66b722ff1 --- /dev/null +++ b/crates/sqlload/src/http_source/main/http_connection_line.rs @@ -0,0 +1,48 @@ +//! Describe the request settings of each `http` connection, as `plan` shows them. + +use std::collections::BTreeMap; + +use crate::http_source::constants::DEFAULT_RETRIES; +use crate::http_source::models::Auth; +use crate::project::models::ConnectionConfig; + +/// One line per `http` connection, such as `http shop rate_limit 10/s, max_in_flight 4, retries 5, auth bearer`: what every request to it shares in a run, how often a failed one is retried, and how it authenticates, never its credentials. +pub fn http_connection_lines(connections: &BTreeMap) -> Vec { + connections + .iter() + .filter_map(|(name, connection)| match connection { + ConnectionConfig::Http { + auth, + rate_limit, + max_in_flight, + retries, + .. + } => Some(format!( + "http {name} rate_limit {}, max_in_flight {}, retries {}, auth {}", + rate_limit.as_deref().unwrap_or("none"), + max_in_flight.map_or_else(|| "none".to_owned(), |most| most.to_string()), + retries.unwrap_or(DEFAULT_RETRIES), + auth.as_ref().map_or_else(|| "none".to_owned(), auth_scheme), + )), + _ => None, + }) + .collect() +} + +/// The connection's `auth` as `plan` names it: `bearer`, `api_key in header X-Api-Key`, `oauth2_password`, ... +fn auth_scheme(auth: &Auth) -> String { + match auth { + Auth::Bearer {} => "bearer".to_owned(), + Auth::ApiKey { + header: Some(header), + .. + } => format!("api_key in header {header}"), + Auth::ApiKey { query, .. } => { + format!("api_key in query {}", query.as_deref().unwrap_or_default()) + } + Auth::Basic {} => "basic".to_owned(), + Auth::Oauth2ClientCredentials(_) => "oauth2_client_credentials".to_owned(), + Auth::Oauth2RefreshToken(_) => "oauth2_refresh_token".to_owned(), + Auth::Oauth2Password(_) => "oauth2_password".to_owned(), + } +} diff --git a/crates/sqlload/src/http_source/main/http_connection_problems.rs b/crates/sqlload/src/http_source/main/http_connection_problems.rs new file mode 100644 index 000000000..334dc3b4d --- /dev/null +++ b/crates/sqlload/src/http_source/main/http_connection_problems.rs @@ -0,0 +1,46 @@ +//! Check the request settings of an `http` connection when the project is read, before any request. + +use crate::http_source::_helpers::endpoint::interval; +use crate::http_source::_helpers::oauth::token_url; +use crate::http_source::constants::{MAX_IN_FLIGHT, MAX_RETRIES}; +use crate::http_source::models::{Auth, ConnectionLimits}; + +/// Why the `auth`, `rate_limit`, `max_in_flight` and `retries` of the `http` connection `name` cannot be used, if they cannot. +pub fn http_connection_problems( + name: &str, + auth: Option<&Auth>, + (limits, retries): (&ConnectionLimits, Option), +) -> Vec { + let place = format!("[connections.{name}]"); + let mut problems: Vec = Vec::new(); + problems.extend(match auth { + Some( + Auth::Oauth2ClientCredentials(spec) + | Auth::Oauth2RefreshToken(spec) + | Auth::Oauth2Password(spec), + ) => token_url(spec).err(), + Some(Auth::ApiKey { header, query }) if header.is_some() == query.is_some() => { + Some("auth api_key takes exactly one of header or query".to_owned()) + } + None | Some(Auth::Bearer {} | Auth::ApiKey { .. } | Auth::Basic {}) => None, + }); + problems.extend( + limits + .rate_limit + .as_deref() + .and_then(|rate| interval(rate).err()), + ); + if limits + .max_in_flight + .is_some_and(|most| !(1..=MAX_IN_FLIGHT).contains(&most)) + { + problems.push(format!("max_in_flight must be from 1 to {MAX_IN_FLIGHT}")); + } + if retries.is_some_and(|retries| retries > MAX_RETRIES) { + problems.push(format!("retries must be at most {MAX_RETRIES}")); + } + problems + .into_iter() + .map(|problem| format!("{place} {problem}")) + .collect() +} diff --git a/crates/sqlload/src/http_source/main/http_credential_source.rs b/crates/sqlload/src/http_source/main/http_credential_source.rs new file mode 100644 index 000000000..a7da5a378 --- /dev/null +++ b/crates/sqlload/src/http_source/main/http_credential_source.rs @@ -0,0 +1,44 @@ +//! Say where an `http` connection's credentials come from, as `sqlload debug` checks them. + +use crate::http_source::models::Auth; + +/// `auth bearer: token from the connection`, or for OAuth2 also the token URL its access tokens come from; an error naming each setting the connection's `auth` needs and lacks. `set` lists which credential settings the connection has, by name. +pub fn http_credential_source(auth: Option<&Auth>, set: &[(&str, bool)]) -> Result { + let (kind, needed, token_url): (&str, &[&str], Option<&str>) = match auth { + None => return Ok("no auth".to_owned()), + Some(Auth::Bearer {}) => ("bearer", &["token"], None), + Some(Auth::ApiKey { .. }) => ("api_key", &["token"], None), + Some(Auth::Basic {}) => ("basic", &["username", "password"], None), + Some(Auth::Oauth2ClientCredentials(spec)) => ( + "oauth2_client_credentials", + &["client_id", "client_secret"], + Some(&spec.token_url), + ), + Some(Auth::Oauth2RefreshToken(spec)) => ( + "oauth2_refresh_token", + &["refresh_token"], + Some(&spec.token_url), + ), + Some(Auth::Oauth2Password(spec)) => ( + "oauth2_password", + &["username", "password"], + Some(&spec.token_url), + ), + }; + let missing: Vec<&str> = needed + .iter() + .copied() + .filter(|key| !set.contains(&(*key, true))) + .collect(); + if !missing.is_empty() { + return Err(format!( + "auth {kind} needs {} on the connection", + missing.join(" and ") + )); + } + let tokens = token_url.map_or_else(String::new, |url| format!("; access tokens from {url}")); + Ok(format!( + "auth {kind}: {} from the connection{tokens}", + needed.join(" and ") + )) +} diff --git a/crates/sqlload/src/http_source/main/mod.rs b/crates/sqlload/src/http_source/main/mod.rs index f48db0ec0..4e09a57aa 100644 --- a/crates/sqlload/src/http_source/main/mod.rs +++ b/crates/sqlload/src/http_source/main/mod.rs @@ -1,2 +1,5 @@ +pub mod http_connection_line; +pub mod http_connection_problems; +pub mod http_credential_source; pub mod literal_path_problems; pub mod run_http_job; diff --git a/crates/sqlload/src/http_source/models.rs b/crates/sqlload/src/http_source/models.rs index 83ed04d71..96031be93 100644 --- a/crates/sqlload/src/http_source/models.rs +++ b/crates/sqlload/src/http_source/models.rs @@ -212,29 +212,45 @@ pub enum Over { FanOut(Box), } -/// The connection a job uses and how it calls it. +/// The connection a job uses, and the headers its own requests add to the connection's. #[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ClientSpec { pub connection: String, #[serde(default)] - pub auth: Auth, - /// Requests allowed per second, minute or hour, such as `10/s`, shared by every job of the connection with the same limit. + pub headers: BTreeMap, +} + +/// What an `http` connection sets for every request to its API in the run: how requests authenticate, the limits they share, and how often a failed one is retried. +#[derive(Debug, Clone, Deserialize)] +pub struct HttpSettings { + /// `None` sends no credential. + #[serde(default)] + pub auth: Option, + #[serde(flatten)] + pub limits: ConnectionLimits, + /// Retries of a request that failed with a transport error, 429 or a 5xx; `None` for [`DEFAULT_RETRIES`](crate::http_source::constants::DEFAULT_RETRIES). + #[serde(default)] + pub retries: Option, +} + +/// The limits every request to one connection shares in the run, across tables, pages, `read_ahead`, fan-out keys and retries. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] +pub struct ConnectionLimits { + /// Requests started per second, minute or hour, such as `10/s`. #[serde(default)] pub rate_limit: Option, - /// Retries of a request that failed with a transport error, 429 or a 5xx. - pub retries: u32, + /// The most requests open at once. #[serde(default)] - pub headers: BTreeMap, + pub max_in_flight: Option, } -/// Credentials from the connection's settings, sent only to the connection's own origin. -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] +/// How requests to the connection authenticate, with credentials from its settings, sent only to its own origin. The variants without settings are empty structs, so a key written beside their `kind` (`{ kind = "bearer", token = "..." }`) is refused rather than dropped. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum Auth { - #[default] - None, /// `Authorization: Bearer `. - Bearer, + Bearer {}, /// The token in a header or a query parameter. ApiKey { #[serde(default)] @@ -243,7 +259,7 @@ pub enum Auth { query: Option, }, /// `username` and `password`. - Basic, + Basic {}, /// An access token from `token_url` for the connection's `client_id` and `client_secret`. Oauth2ClientCredentials(OAuth2Spec), /// An access token from `token_url` for the connection's `refresh_token`. @@ -253,7 +269,8 @@ pub enum Auth { } /// Where an OAuth2 access token comes from and how the client authenticates there. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct OAuth2Spec { pub token_url: String, #[serde(default)] @@ -444,6 +461,8 @@ pub struct Calls { pub http: reqwest::Client, pub endpoint: Endpoint, pub limiter: Option>, + /// The connection's `max_in_flight`: each attempt holds a permit while its request is open. + pub in_flight: Option>, pub retries: u32, pub on_status: BTreeMap, /// Whether an offset page of this request held its whole limit in this job, which shows the API does not cap its pages lower, so a short first page needs no check; shared by its clones, the keys of a fan-out. diff --git a/crates/sqlload/src/http_source/tests.rs b/crates/sqlload/src/http_source/tests.rs index d878a1ced..ab7c531b5 100644 --- a/crates/sqlload/src/http_source/tests.rs +++ b/crates/sqlload/src/http_source/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_credential_source.rs"] +mod credential_source; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_job_problems.rs"] diff --git a/crates/sqlload/src/http_source/tests/test_credential_source.rs b/crates/sqlload/src/http_source/tests/test_credential_source.rs new file mode 100644 index 000000000..a222ce2a5 --- /dev/null +++ b/crates/sqlload/src/http_source/tests/test_credential_source.rs @@ -0,0 +1,65 @@ +use crate::http_source::main::http_credential_source::http_credential_source; +use crate::http_source::models::Auth; +use crate::http_source::tests::test_types::CredentialSourceTestCase; + +#[test] +fn given_an_http_connections_auth_when_debugging_then_it_names_the_settings_the_credentials_come_from() + { + let test_cases = [ + CredentialSourceTestCase { + description: "bearer with its token", + auth: Some(r#"kind = "bearer""#), + set: &["token", "username"], + expected_source: Ok("auth bearer: token from the connection"), + }, + CredentialSourceTestCase { + description: "OAuth2 also names the token URL", + auth: Some( + r#"kind = "oauth2_client_credentials" +token_url = "https://auth.example.com/token""#, + ), + set: &["client_id", "client_secret"], + expected_source: Ok( + "auth oauth2_client_credentials: client_id and client_secret from the connection; access tokens from https://auth.example.com/token", + ), + }, + CredentialSourceTestCase { + description: "basic without a password names only what is missing", + auth: Some(r#"kind = "basic""#), + set: &["username", "token"], + expected_source: Err("auth basic needs password on the connection"), + }, + CredentialSourceTestCase { + description: "no auth needs nothing", + auth: None, + set: &[], + expected_source: Ok("no auth"), + }, + ]; + for test_case in test_cases { + let auth: Option = test_case + .auth + .map(toml::from_str::) + .transpose() + .unwrap_or_else(|error| panic!("{}: {error}", test_case.description)); + let set: Vec<(&str, bool)> = [ + "token", + "username", + "password", + "client_id", + "client_secret", + ] + .into_iter() + .map(|key| (key, test_case.set.contains(&key))) + .collect(); + assert_eq!( + http_credential_source(auth.as_ref(), &set), + test_case + .expected_source + .map(str::to_owned) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/http_source/tests/test_job_problems.rs b/crates/sqlload/src/http_source/tests/test_job_problems.rs index de33366a2..bddb6ba2a 100644 --- a/crates/sqlload/src/http_source/tests/test_job_problems.rs +++ b/crates/sqlload/src/http_source/tests/test_job_problems.rs @@ -8,35 +8,34 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an let test_cases = [ JobProblemsTestCase { description: "a paginate with a next link and a selector is fine", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":3,"rate_limit":"10/s"},"path":"/orders","paginator":{"kind":"next_link","pointer":"$.next"},"select":"$.items"}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"next_link","pointer":"$.next"},"select":"$.items"}"#, expected_problems: &[], }, JobProblemsTestCase { - description: "a bad selector, rate and status are all reported", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0,"rate_limit":"fast"},"path":"/orders","paginator":{"kind":"single"},"select":"items","on_status":{"999":"skip"}}"#, + description: "a bad selector and status are both reported", + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"items","on_status":{"999":"skip"}}"#, expected_problems: &[ "select \"items\" is not a path such as \"$\", \"$.items\" or \"$.items[*].id\"; use dotted field names after \"$\"", - "rate_limit \"fast\" is not a rate such as \"10/s\", \"600/m\" or \"1000/h\"", "on_status 999 is not an HTTP status", ], }, JobProblemsTestCase { description: "a paginate path with a placeholder needs fan_out", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders/{id}","paginator":{"kind":"single"}}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/orders/{id}","paginator":{"kind":"single"}}"#, expected_problems: &[ "paginate path /orders/{id} has the placeholder {id}; use fan_out to fill it", ], }, JobProblemsTestCase { description: "a fan-out over a paginate whose field does not fill the placeholder", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/orders/{order_id}/items","paginator":{"kind":"single"},"concurrency":4,"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/orders/{order_id}/items","paginator":{"kind":"single"},"concurrency":4,"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, expected_problems: &[ "fan-out path /orders/{order_id}/items needs {order_id}, but over gives only id", ], }, JobProblemsTestCase { description: "a fan-out over a paginate without a key field, and zero concurrency", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/orders/{id}","paginator":{"kind":"single"},"concurrency":0,"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"select":"$.items"}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/orders/{id}","paginator":{"kind":"single"},"concurrency":0,"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"$.items"}}"#, expected_problems: &[ "fan_out concurrency must be at least 1", "fan_out over a paginate needs a select naming each key's field, such as \"$.items[*].id\", not \"$.items\"", @@ -44,33 +43,33 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "listed keys must name every placeholder of a path with two", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/{region}/orders/{id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"values","values":[{"region":"uk"}]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/{region}/orders/{id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"values","values":[{"region":"uk"}]}}"#, expected_problems: &[ "fan-out path /{region}/orders/{id} needs {id}, which the key {\"region\":\"uk\"} lacks", ], }, JobProblemsTestCase { description: "a third level may use the keys of both levels above it", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{order_id}/i/{item_id}/n","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{order_id}/i","paginator":{"kind":"single"},"select":"$.items[*].item_id","concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].order_id"}}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{order_id}/i/{item_id}/n","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{order_id}/i","paginator":{"kind":"single"},"select":"$.items[*].item_id","concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].order_id"}}}"#, expected_problems: &[], }, JobProblemsTestCase { description: "a third level's placeholder that no level above gives", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{order_id}/i/{item_id}/n/{customer_id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{order_id}/i","paginator":{"kind":"single"},"select":"$.items[*].item_id","concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].order_id"}}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{order_id}/i/{item_id}/n/{customer_id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{order_id}/i","paginator":{"kind":"single"},"select":"$.items[*].item_id","concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].order_id"}}}"#, expected_problems: &[ "fan-out path /o/{order_id}/i/{item_id}/n/{customer_id} needs {customer_id}, but over gives only order_id, item_id", ], }, JobProblemsTestCase { description: "a level that picks a key name a level above already gives", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{order_id}/n","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{order_id}/i","paginator":{"kind":"single"},"select":"$.items[*].order_id","concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].order_id"}}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{order_id}/n","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{order_id}/i","paginator":{"kind":"single"},"select":"$.items[*].order_id","concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].order_id"}}}"#, expected_problems: &[ "fan_out over picks order_id, which a level above already gives; name it apart with key=", ], }, JobProblemsTestCase { description: "a pointer that is not a path, a zero limit and a bad header", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0,"headers":{"X Bad":"1"}},"path":"/o","paginator":{"kind":"next_link","pointer":"next"}}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop","headers":{"X Bad":"1"}},"path":"/o","paginator":{"kind":"next_link","pointer":"next"}}"#, expected_problems: &[ "paginator pointer \"next\" is not a path such as \"$.next\" or \"$.meta.next_cursor\"", "header \"X Bad\" is not a valid HTTP header name and value", @@ -78,31 +77,31 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "an offset limit of zero", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"offset","param":"offset","limit_param":"limit","limit":0}}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"offset","param":"offset","limit_param":"limit","limit":0}}"#, expected_problems: &["sl.offset limit must be at least 1"], }, JobProblemsTestCase { description: "a page-number read-ahead of zero pages", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"page_number","param":"p","start":1,"read_ahead":0}}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"page_number","param":"p","start":1,"read_ahead":0}}"#, expected_problems: &["read_ahead must be at least 1 page"], }, JobProblemsTestCase { description: "a read-ahead of more pages than the cap", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"offset","param":"offset","limit_param":"limit","limit":10,"read_ahead":101}}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"offset","param":"offset","limit_param":"limit","limit":10,"read_ahead":101}}"#, expected_problems: &[ "read_ahead 101 is more than 100 pages at once; most APIs refuse that many open requests", ], }, JobProblemsTestCase { description: "a cursor pointer through a list", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"cursor","pointer":"$.meta[*].next","param":"after"}}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"cursor","pointer":"$.meta[*].next","param":"after"}}"#, expected_problems: &[ "paginator pointer \"$.meta[*].next\" is not a path such as \"$.next\" or \"$.meta.next_cursor\"", ], }, JobProblemsTestCase { description: "key_path on a fan-out that feeds keys to the level below, and key_path at the whole record", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"key_path":{"path":"$"},"over":{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/p/{x}","paginator":{"kind":"single"},"select":"$.items[*].id","concurrency":1,"key_path":{"path":"$.x"},"over":{"kind":"values","values":[1]}}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"key_path":{"path":"$"},"over":{"kind":"fan_out","client":{"connection":"shop"},"path":"/p/{x}","paginator":{"kind":"single"},"select":"$.items[*].id","concurrency":1,"key_path":{"path":"$.x"},"over":{"kind":"values","values":[1]}}}"#, expected_problems: &[ "fan_out's over picks keys with its select; key_path checks the records of the fan-out that loads rows", "key_path \"$\" is not a path such as \"$.issueId\"; use dotted field names after \"$\"", @@ -110,19 +109,19 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "key_path on a request filling two placeholders, one in the path and one in json_body", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/orgs/{org}/issues","method":"POST","json_body":{"ids":["{id}"]},"paginator":{"kind":"single"},"concurrency":1,"key_path":{"path":"$.issueId"},"over":{"kind":"values","values":[{"org":1,"id":2}]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/orgs/{org}/issues","method":"POST","json_body":{"ids":["{id}"]},"paginator":{"kind":"single"},"concurrency":1,"key_path":{"path":"$.issueId"},"over":{"kind":"values","values":[{"org":1,"id":2}]}}"#, expected_problems: &[ "key_path checks each record against the one key its request sent, but the request fills 2 placeholders (org, id), so a record could match the wrong one; fan out over one placeholder", ], }, JobProblemsTestCase { description: "indexes in sl.parent and key_path are fine", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"select":"$.issues[*].histories[*]","fields":[{"name":"last","path":"$.changes[-1].id","from":{"kind":"parent","levels":1}},{"name":"first","path":"$.items[0].to"}],"key_path":{"path":"$.keys[0]","from":{"kind":"parent","levels":1}},"over":{"kind":"values","values":[1]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"select":"$.issues[*].histories[*]","fields":[{"name":"last","path":"$.changes[-1].id","from":{"kind":"parent","levels":1}},{"name":"first","path":"$.items[0].to"}],"key_path":{"path":"$.keys[0]","from":{"kind":"parent","levels":1}},"over":{"kind":"values","values":[1]}}"#, expected_problems: &[], }, JobProblemsTestCase { description: "unsupported syntax in sl.parent and key_path is refused, naming the field and the path", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"select":"$.issues[*].histories[*]","fields":[{"name":"first","path":"$.items[0].to"},{"name":"all","path":"$.items[*].id","from":{"kind":"parent","levels":1}}],"key_path":{"path":"$.keys[0:1]","from":{"kind":"parent","levels":1}},"over":{"kind":"values","values":[1]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"select":"$.issues[*].histories[*]","fields":[{"name":"first","path":"$.items[0].to"},{"name":"all","path":"$.items[*].id","from":{"kind":"parent","levels":1}}],"key_path":{"path":"$.keys[0:1]","from":{"kind":"parent","levels":1}},"over":{"kind":"values","values":[1]}}"#, expected_problems: &[ "fields[\"all\"]: field path \"$.items[*].id\" uses the wildcard [*], which one column can't hold; take one item with [n] or [-1], or select the list's items as rows", "key_path \"$.keys[0:1]\" uses the slice [0:1], which field paths don't support; they take dotted names and [n] or [-1] indexes, such as \"$.items[0].id\"", @@ -130,35 +129,35 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "a paginate selecting a key field", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].id"}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].id"}"#, expected_problems: &[ "select \"$.items[*].id\" names a key field (id); that form is for fan_out's over, so select the rows instead", ], }, JobProblemsTestCase { description: "listed values over a path without a placeholder", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"values","values":[1,2]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"values","values":[1,2]}}"#, expected_problems: &[ "fan-out path /orders has no placeholder, so every key would send the same request; add one such as {id} to the path or json_body", ], }, JobProblemsTestCase { description: "a fan-out whose own select names a key field", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/orders/{id}","paginator":{"kind":"single"},"select":"$.items[*].sku","concurrency":2,"over":{"kind":"values","values":[1]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/orders/{id}","paginator":{"kind":"single"},"select":"$.items[*].sku","concurrency":2,"over":{"kind":"values","values":[1]}}"#, expected_problems: &[ "select \"$.items[*].sku\" names a key field (sku); that form is for fan_out's over, so select the rows instead", ], }, JobProblemsTestCase { description: "fields on a fan-out's over are refused; they belong to the fan-out's rows", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/orders/{id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id","fields":[{"name":"id","path":"$.id"}]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/orders/{id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id","fields":[{"name":"id","path":"$.id"}]}}"#, expected_problems: &[ "fan_out's over picks keys with its select; fields and keep_record are for the fan-out's own rows", ], }, JobProblemsTestCase { description: "a one-level fan-out's own select problem comes before its listed key's, as before multi-level fan-out", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/{region}/orders/{id}","paginator":{"kind":"single"},"select":"$.items[*].sku","concurrency":2,"over":{"kind":"values","values":[{"region":"uk"}]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/{region}/orders/{id}","paginator":{"kind":"single"},"select":"$.items[*].sku","concurrency":2,"over":{"kind":"values","values":[{"region":"uk"}]}}"#, expected_problems: &[ "select \"$.items[*].sku\" names a key field (sku); that form is for fan_out's over, so select the rows instead", "fan-out path /{region}/orders/{id} needs {id}, which the key {\"region\":\"uk\"} lacks", @@ -166,7 +165,7 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "a one-level over's fields problem comes before its placeholder problem, as before multi-level fan-out", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/orders/{id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/shops/{shop}/orders","paginator":{"kind":"single"},"select":"$.items[*].id","keep_record":"raw"}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/orders/{id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/shops/{shop}/orders","paginator":{"kind":"single"},"select":"$.items[*].id","keep_record":"raw"}}"#, expected_problems: &[ "fan_out's over picks keys with its select; fields and keep_record are for the fan-out's own rows", "fan_out over path /shops/{shop}/orders has the placeholder {shop}; over must be a plain paginate", @@ -174,24 +173,24 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "body placeholders filled from a paginate's keys; a GraphQL query's braces stay text", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"json_body":{"query":"{ order { lines } }","order":{"id":"{id}"}},"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"json_body":{"query":"{ order { lines } }","order":{"id":"{id}"}},"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, expected_problems: &[], }, JobProblemsTestCase { description: "a body placeholder no level above gives", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"json_body":{"order":{"id":"{order_id}"}},"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"json_body":{"order":{"id":"{order_id}"}},"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, expected_problems: &["fan-out json_body needs {order_id}, but over gives only id"], }, JobProblemsTestCase { description: "a listed key lacking a body placeholder", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"json_body":{"region":"{region}","id":"{id}"},"over":{"kind":"values","values":[{"id":1}]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"json_body":{"region":"{region}","id":"{id}"},"over":{"kind":"values","values":[{"id":1}]}}"#, expected_problems: &[ "fan-out json_body needs {region}, which the key {\"id\":1} lacks", ], }, JobProblemsTestCase { description: "a batch of zero, with a path placeholder and a placeholder within other text", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/lookup/{id}","method":"POST","paginator":{"kind":"single"},"concurrency":2,"batch":0,"json_body":{"q":"id in {id}"},"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/lookup/{id}","method":"POST","paginator":{"kind":"single"},"concurrency":2,"batch":0,"json_body":{"q":"id in {id}"},"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}"#, expected_problems: &[ "fan_out batch must be at least 1", "a batched fan-out sends many keys in one request, so its path /lookup/{id} cannot have the placeholder {id}; put the keys in json_body", @@ -200,7 +199,7 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "a batch without a body to put its keys in", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"batch":100,"over":{"kind":"values","values":[1]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"batch":100,"over":{"kind":"values","values":[1]}}"#, expected_problems: &[ "fan-out path /lookup has no placeholder, so every key would send the same request; add one such as {id} to the path or json_body", "fan_out batch= sends each batch's keys as a list in json_body, so it needs json_body with a placeholder, such as {\"ids\": \"{id}\"}", @@ -208,19 +207,19 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "a batched level used as another fan-out's over", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{id}/l/{line_id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"batch":10,"select":"$.items[*].line_id","json_body":{"ids":"{id}"},"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{id}/l/{line_id}","paginator":{"kind":"single"},"concurrency":2,"over":{"kind":"fan_out","client":{"connection":"shop"},"path":"/lookup","method":"POST","paginator":{"kind":"single"},"concurrency":2,"batch":10,"select":"$.items[*].line_id","json_body":{"ids":"{id}"},"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/orders","paginator":{"kind":"single"},"select":"$.items[*].id"}}}"#, expected_problems: &[ "batch= is for the fan-out that loads rows; a fan_out used as over sends one request per key", ], }, JobProblemsTestCase { description: "cells named by a listed column, by the header row and by each cell itself are fine, and so is an error_path", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/grid/{id}","paginator":{"kind":"single"},"select":"$.rows","concurrency":1,"error_path":"$.Result.ErrorAlert","cells":{"pointer":"$.DATAFIELD","value":["$.FIELDVALUE"],"name":{"kind":"field","path":"$.FIELDNAME"}},"over":{"kind":"values","values":[{"id":1}]}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/grid/{id}","paginator":{"kind":"single"},"select":"$.rows","concurrency":1,"error_path":"$.Result.ErrorAlert","cells":{"pointer":"$.DATAFIELD","value":["$.FIELDVALUE"],"name":{"kind":"field","path":"$.FIELDNAME"}},"over":{"kind":"values","values":[{"id":1}]}}"#, expected_problems: &[], }, JobProblemsTestCase { description: "a listed column source that is not a path, and a self-named cell's name path through a list", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/grid/{id}","paginator":{"kind":"single"},"select":"$.rows","concurrency":1,"cells":{"pointer":"$.cells","value":["$.value"],"name":{"kind":"field","path":"$.names[*].text"}},"over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/sheets","paginator":{"kind":"single"},"select":"$.rows[*].id","cells":{"pointer":"$.cells","value":["$.value"],"name":{"kind":"listed","from":"columns","key":"columnId","id_key":"id","name_key":"title"}}}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/grid/{id}","paginator":{"kind":"single"},"select":"$.rows","concurrency":1,"cells":{"pointer":"$.cells","value":["$.value"],"name":{"kind":"field","path":"$.names[*].text"}},"over":{"kind":"paginate","client":{"connection":"shop"},"path":"/sheets","paginator":{"kind":"single"},"select":"$.rows[*].id","cells":{"pointer":"$.cells","value":["$.value"],"name":{"kind":"listed","from":"columns","key":"columnId","id_key":"id","name_key":"title"}}}}"#, expected_problems: &[ "cells path \"$.names[*].text\" cannot pick from a list with [*]", "cells path \"columns\" is not a path such as \"$.cells\" or \"$.value\"", @@ -229,14 +228,14 @@ fn given_a_job_definition_when_checking_it_then_every_problem_is_found_before_an }, JobProblemsTestCase { description: "a header-row cells pointer that is not one field of the row", - job: r#"{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/grid","paginator":{"kind":"single"},"select":"$.rows","cells":{"pointer":"$","value":["$.v"],"name":{"kind":"header_row"}}}"#, + job: r#"{"kind":"paginate","client":{"connection":"shop"},"path":"/grid","paginator":{"kind":"single"},"select":"$.rows","cells":{"pointer":"$","value":["$.v"],"name":{"kind":"header_row"}}}"#, expected_problems: &[ "cells pointer \"$\" must name one field of the row, such as \"$.cells\"", ], }, JobProblemsTestCase { description: "an error_path at the whole response, and one through a list on the level that feeds keys", - job: r#"{"kind":"fan_out","client":{"connection":"shop","retries":0},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"error_path":"$","over":{"kind":"paginate","client":{"connection":"shop","retries":0},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].id","error_path":"$.errors[*].message"}}"#, + job: r#"{"kind":"fan_out","client":{"connection":"shop"},"path":"/o/{id}","paginator":{"kind":"single"},"concurrency":1,"error_path":"$","over":{"kind":"paginate","client":{"connection":"shop"},"path":"/o","paginator":{"kind":"single"},"select":"$.items[*].id","error_path":"$.errors[*].message"}}"#, expected_problems: &[ "error_path \"$\" is not a path such as \"$.errors\" or \"$.result.error\"; name one value, without [*]", "error_path \"$.errors[*].message\" is not a path such as \"$.errors\" or \"$.result.error\"; name one value, without [*]", diff --git a/crates/sqlload/src/http_source/tests/test_types.rs b/crates/sqlload/src/http_source/tests/test_types.rs index b59b82621..97bdd3d9e 100644 --- a/crates/sqlload/src/http_source/tests/test_types.rs +++ b/crates/sqlload/src/http_source/tests/test_types.rs @@ -23,3 +23,12 @@ pub(crate) struct LimiterTestCase { /// How many requests went. pub(crate) expected_turns: usize, } + +pub(crate) struct CredentialSourceTestCase { + pub(crate) description: &'static str, + /// The connection's `auth`, as TOML; `None` when it sets none. + pub(crate) auth: Option<&'static str>, + /// The credential settings the connection has. + pub(crate) set: &'static [&'static str], + pub(crate) expected_source: Result<&'static str, &'static str>, +} diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index 72109be2c..0ef9f99ad 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -6,6 +6,7 @@ use crate::adapters::dispatch::main::session::live_columns::live_columns; use crate::adapters::dispatch::models::Destination; use crate::column_types::models::{ColumnType, Warehouse}; use crate::files_source::models::{AfterLoad, OnEmpty}; +use crate::http_source::main::http_connection_line::http_connection_lines; use crate::load_events::main::begin_step::begin_step; use crate::load_events::main::end_step::end_step; use crate::load_events::models::StepEnd; @@ -1575,6 +1576,7 @@ pub(crate) async fn found_sources<'a>( options: &PlanOptions, ) -> Result, LoaderError> { let mut found = FoundSources::default(); + found.plan.http = http_connection_lines(&project.connections); let mut drained_sources: Vec<(usize, DrainedFiles)> = Vec::new(); for (source_name, source) in &project.sources { begin_step(source_name, "listing tables"); diff --git a/crates/sqlload/src/loader/main/debug_project.rs b/crates/sqlload/src/loader/main/debug_project.rs index d99201e4f..64383f3a5 100644 --- a/crates/sqlload/src/loader/main/debug_project.rs +++ b/crates/sqlload/src/loader/main/debug_project.rs @@ -2,12 +2,14 @@ use crate::adapters::dispatch::main::session::check_destination::check_destination; use crate::adapters::dispatch::main::session::connect_destination::connect_destination; +use crate::auth::models::Secret; use crate::file_store::main::gcs_key_source::gcs_key_source; use crate::file_store::main::s3_key_source::s3_key_source; +use crate::http_source::main::http_credential_source::http_credential_source; use crate::loader::main::_check_source::check_source; use crate::project::models::{ConnectionConfig, ProjectConfig}; -/// One line per check (`ok`, `warn` or `fail`), and whether nothing failed: where each S3 connection's keys and each GCS connection's credentials come from, then each target and source; `connect = false` skips them. +/// One line per check (`ok`, `warn` or `fail`), and whether nothing failed: where each S3 connection's keys, each GCS connection's credentials and each HTTP connection's credentials come from, then each target and source; `connect = false` skips them. pub async fn debug_project(project: &ProjectConfig, connect: bool) -> (Vec, bool) { let mut lines = vec![format!( "ok config {} connections, {} targets, {} sources", @@ -42,6 +44,29 @@ pub async fn debug_project(project: &ProjectConfig, connect: bool) -> (Vec gcs_key_source(service_account_key.is_some()) .map(|source| format!("GCS credentials from {source}")), + ConnectionConfig::Http { + auth, + token, + username, + password, + client_id, + client_secret, + refresh_token, + .. + } => { + let secret = + |value: &Option| value.as_ref().is_some_and(|it| !it.0.is_empty()); + let text = |value: &Option| value.as_ref().is_some_and(|it| !it.is_empty()); + let set = [ + ("token", secret(token)), + ("username", text(username)), + ("password", secret(password)), + ("client_id", text(client_id)), + ("client_secret", secret(client_secret)), + ("refresh_token", secret(refresh_token)), + ]; + http_credential_source(auth.as_ref(), &set) + } _ => continue, }; lines.push(match found { diff --git a/crates/sqlload/src/loader/main/render_plan.rs b/crates/sqlload/src/loader/main/render_plan.rs index 837b93968..1dfeb570a 100644 --- a/crates/sqlload/src/loader/main/render_plan.rs +++ b/crates/sqlload/src/loader/main/render_plan.rs @@ -8,12 +8,15 @@ use crate::loader::models::{PlanItem, ProjectPlan, TablePlan}; use crate::plan_text::main::plan_lines::plan_lines; use crate::plan_text::models::{LoadedTable, PlanParts}; -/// The plan's lines, and whether every selected table can load: only the lines a table has of its own or that ask for something (see [`plan_lines`]), or with `verbose` every line; a plan of only project problems shows just its `Errors`, and tables stopped for the same reasons share one item. +/// The plan's lines, and whether every selected table can load: each `http` connection's settings, then only the lines a table has of its own or that ask for something (see [`plan_lines`]), or with `verbose` every line; a plan of only project problems shows just its `Errors`, and tables stopped for the same reasons share one item. pub fn render_plan(plan: &ProjectPlan, verbose: bool) -> (Vec, bool) { let mut view = text_view(plan); view.errors = merged_errors(std::mem::take(&mut view.errors)); if verbose { - return (view_lines(&view), view.ok); + return ( + with_http_lines(view_lines(&view), &plan.http, true), + view.ok, + ); } let ready: Vec<&TablePlan> = plan .tables @@ -39,5 +42,19 @@ pub fn render_plan(plan: &ProjectPlan, verbose: bool) -> (Vec, bool) { tables, problems, }; - (plan_lines(&parts), view.ok) + ( + with_http_lines(plan_lines(&parts), &plan.http, false), + view.ok, + ) +} + +/// `lines` with each `http` connection's line: under `-v` (`verbose`) after the `Plan ready`, `to` and `memory` lines, otherwise first, apart from the rest. +fn with_http_lines(mut lines: Vec, http: &[String], verbose: bool) -> Vec { + let at = match verbose { + true => lines.iter().position(String::is_empty).unwrap_or_default(), + false => 0, + }; + let gap = (!verbose && !http.is_empty()).then(String::new); + lines.splice(at..at, http.iter().cloned().chain(gap)); + lines } diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index bab8c3bd4..4df35dfc5 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -681,6 +681,8 @@ pub struct ProjectPlan { pub memory: Option, /// The `memory` line asks for something, so `plan` shows it without `-v` too (see [`RunMemory::asks`](crate::memory_budget::models::RunMemory)). pub memory_asks: bool, + /// One line per `http` connection: the limits every request to it shares in the run, its retries and its auth scheme. + pub http: Vec, } /// The tables a source that could not be read would load, as far as its settings name them. diff --git a/crates/sqlload/src/project/_helpers/references.rs b/crates/sqlload/src/project/_helpers/references.rs index b5b862fcd..6eba0c70a 100644 --- a/crates/sqlload/src/project/_helpers/references.rs +++ b/crates/sqlload/src/project/_helpers/references.rs @@ -5,6 +5,8 @@ use crate::files_source::main::text_charset::text_charset; use crate::files_source::models::{ AfterLoad, Compression, CsvQuote, FilePattern, FilesOrder, OnEmpty, RecordShape, SourceFormat, }; +use crate::http_source::main::http_connection_problems::http_connection_problems; +use crate::http_source::models::ConnectionLimits; use crate::load_naming::constants::SQLLOAD_PREFIX; use crate::project::_helpers::targets::{ destination_kind_problems, is_files_target, stage_problems, target_problems, @@ -52,6 +54,24 @@ pub(crate) fn reference_problems(project: &ProjectConfig) -> Vec { "[connections.{name}] profile is empty; omit it to use AWS_PROFILE, else default" )); } + if let ConnectionConfig::Http { + auth, + rate_limit, + max_in_flight, + retries, + .. + } = connection + { + let limits = ConnectionLimits { + rate_limit: rate_limit.clone(), + max_in_flight: *max_in_flight, + }; + problems.extend(http_connection_problems( + name, + auth.as_ref(), + (&limits, *retries), + )); + } if let ConnectionConfig::Gcs { hmac_key_id, hmac_secret, diff --git a/crates/sqlload/src/project/_helpers/serde_values.rs b/crates/sqlload/src/project/_helpers/serde_values.rs index 6a3a3fa6b..255a3d758 100644 --- a/crates/sqlload/src/project/_helpers/serde_values.rs +++ b/crates/sqlload/src/project/_helpers/serde_values.rs @@ -2,9 +2,25 @@ use std::time::Duration; use serde::{Deserialize, Deserializer}; +use crate::http_source::models::Auth; use crate::project::constants::{MAX_PERCENT, WHOLE_BATCH}; use crate::project::models::{BatchStep, Lookback}; +/// Deserialize an `http` connection's optional `auth`, a table such as `{ kind = "bearer" }`. Any other value fails without showing it, as a mistaken `auth = "${ENV:API_TOKEN}"` would otherwise print the token. +pub(crate) fn deserialize_optional_auth<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + match Option::::deserialize(deserializer)? { + None => Ok(None), + Some(table @ toml::Value::Table(_)) => Auth::deserialize(table) + .map(Some) + .map_err(serde::de::Error::custom), + Some(_) => Err(serde::de::Error::custom( + "auth is a table such as { kind = \"bearer\" }; credentials such as token go in their own keys", + )), + } +} + /// Deserialize an optional duration written as text such as `1h` or `2d`. pub(crate) fn deserialize_optional_duration<'de, D: Deserializer<'de>>( deserializer: D, diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index a63f72be9..1c33e40eb 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -14,8 +14,8 @@ use crate::http_source::models::LiteralPaths; use crate::load_naming::models::{Compat, Naming, OnDuplicateNames}; use crate::project::_helpers::defaults; use crate::project::_helpers::serde_values::{ - deserialize_optional_batch, deserialize_optional_duration, deserialize_optional_keys, - deserialize_optional_lookback, deserialize_optional_percent, + deserialize_optional_auth, deserialize_optional_batch, deserialize_optional_duration, + deserialize_optional_keys, deserialize_optional_lookback, deserialize_optional_percent, }; use crate::project::constants::{ DEFAULT_DESTINATION_PATH, DEFAULT_FILES_TABLE_NAME, DEFAULT_MAX_FOOTER_READS, @@ -311,26 +311,38 @@ pub enum ConnectionConfig { #[serde(default)] token_url: Option, }, - /// An HTTP API, for Python sources; its settings reach table functions as `ctx.connection`. + /// An HTTP API, for Python sources' `sl.http` jobs; its settings reach table functions as `ctx.connection`. Http { base_url: String, + /// How requests authenticate, with the credentials below; `None` sends none. + #[serde(default, deserialize_with = "deserialize_optional_auth")] + auth: Option, #[serde(default)] token: Option, - /// For `sl.basic()` and `sl.oauth2_password(...)`. + /// For `auth` `basic` and `oauth2_password`. #[serde(default)] username: Option, #[serde(default)] password: Option, - /// For `sl.oauth2_client_credentials(...)`, and `sl.oauth2_refresh_token(...)` or `sl.oauth2_password(...)` when the API wants the client too. + /// For `auth` `oauth2_client_credentials`, and `oauth2_refresh_token` or `oauth2_password` when the API wants the client too. #[serde(default)] client_id: Option, #[serde(default)] client_secret: Option, - /// For `sl.oauth2_refresh_token(...)`. + /// For `auth` `oauth2_refresh_token`. #[serde(default)] refresh_token: Option, #[serde(default)] headers: BTreeMap, + /// Requests started per second, minute or hour (`10/s`), shared by every request to the connection in the run. + #[serde(default)] + rate_limit: Option, + /// The most requests open at once on the connection in the run. + #[serde(default)] + max_in_flight: Option, + /// Retries of a request that failed with a transport error, 429 or a 5xx. + #[serde(default)] + retries: Option, }, } diff --git a/crates/sqlload/src/project/tests/test_parse_project.rs b/crates/sqlload/src/project/tests/test_parse_project.rs index 679801b76..f94b2d0d3 100644 --- a/crates/sqlload/src/project/tests/test_parse_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_project.rs @@ -484,6 +484,61 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { text: "[connections.lake]\nkind = \"gcs\"\nbucket = \"b\"\nhmac_key_id = \"GOOG1E\"\n", expected_message: "[connections.lake] hmac_key_id and hmac_secret go together", }, + ParseProjectErrorTestCase { + description: "an http connection's rate is checked when the project is read", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nrate_limit = \"fast\"\n", + expected_message: "[connections.api] rate_limit \"fast\" is not a rate such as \"10/s\", \"600/m\" or \"1000/h\"", + }, + ParseProjectErrorTestCase { + description: "an in-flight cap of none would never send a request", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nmax_in_flight = 0\n", + expected_message: "[connections.api] max_in_flight must be from 1 to 10000", + }, + ParseProjectErrorTestCase { + description: "an API key goes in a header or a query parameter, not both", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nauth = { kind = \"api_key\", header = \"X-Key\", query = \"key\" }\n", + expected_message: "[connections.api] auth api_key takes exactly one of header or query", + }, + ParseProjectErrorTestCase { + description: "a token URL the client secret would reach in clear text is refused", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nauth = { kind = \"oauth2_client_credentials\", token_url = \"http://auth.example.com/token\" }\n", + expected_message: "[connections.api] token_url http://auth.example.com/token must be https", + }, + ParseProjectErrorTestCase { + description: "a misspelt auth setting is never silently ignored", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nauth = { kind = \"oauth2_password\", token_url = \"https://a.example.com/t\", scopes = \"x\" }\n", + expected_message: "unknown field `scopes`", + }, + ParseProjectErrorTestCase { + description: "an unknown auth scheme is refused", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nauth = { kind = \"digest\" }\n", + expected_message: "unknown variant `digest`", + }, + ParseProjectErrorTestCase { + description: "a rate limit on a connection that sends no HTTP request is refused", + text: "[connections.sf]\nkind = \"snowflake\"\naccount = \"a\"\nrate_limit = \"10/s\"\n", + expected_message: "unknown field `rate_limit`", + }, + ParseProjectErrorTestCase { + description: "a token written as auth is refused without showing it", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nauth = \"${ENV:API_TOKEN}\"\n", + expected_message: "auth is a table such as { kind = \"bearer\" }; credentials such as token go in their own keys", + }, + ParseProjectErrorTestCase { + description: "a credential written inside a bearer auth is never silently dropped", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nauth = { kind = \"bearer\", token = \"t\" }\n", + expected_message: "unknown field `token`", + }, + ParseProjectErrorTestCase { + description: "an in-flight cap past what one process can hold is refused", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nmax_in_flight = 1000000000000\n", + expected_message: "[connections.api] max_in_flight must be from 1 to 10000", + }, + ParseProjectErrorTestCase { + description: "a retry count that would overflow is refused", + text: "[connections.api]\nkind = \"http\"\nbase_url = \"https://api.example.com\"\nretries = 4294967295\n", + expected_message: "[connections.api] retries must be at most 100", + }, ParseProjectErrorTestCase { description: "an odbc connection needs a connection string or a DSN", text: "[connections.erp]\nkind = \"odbc\"\n", diff --git a/packages/sqlload/python/sqlload/__init__.py b/packages/sqlload/python/sqlload/__init__.py index dfbcdefea..f69b3c6e6 100644 --- a/packages/sqlload/python/sqlload/__init__.py +++ b/packages/sqlload/python/sqlload/__init__.py @@ -34,9 +34,6 @@ def orders(ctx: sl.Context): Cells, Http, Job, - api_key, - basic, - bearer, cells, cursor, header_row, @@ -44,9 +41,6 @@ def orders(ctx: sl.Context): link_header, listed, next_link, - oauth2_client_credentials, - oauth2_password, - oauth2_refresh_token, offset, page_number, single, @@ -69,9 +63,6 @@ def orders(ctx: sl.Context): "Table", "TransformContext", "__version__", - "api_key", - "basic", - "bearer", "cells", "checkpoint", "cursor", @@ -85,9 +76,6 @@ def orders(ctx: sl.Context): "listed", "ndjson", "next_link", - "oauth2_client_credentials", - "oauth2_password", - "oauth2_refresh_token", "offset", "page_number", "parent", diff --git a/packages/sqlload/python/sqlload/_http.py b/packages/sqlload/python/sqlload/_http.py index ed3d38a42..f7908c65a 100644 --- a/packages/sqlload/python/sqlload/_http.py +++ b/packages/sqlload/python/sqlload/_http.py @@ -14,8 +14,8 @@ from sqlload._declare import Source from sqlload._transform import TRANSFORMS -#: Default retries of a request that failed with a transport error, 429 or a 5xx. -DEFAULT_RETRIES = 5 +#: What moved from `sl.http` onto the `http` connection in sqlload.toml. +CONNECTION_SETTINGS = ("auth", "rate_limit", "max_in_flight", "retries") ROW_LOOP_HINT = ( "rows of an sl.http job stay in Rust and are never handed to Python one at a time (rule python_row_loop); " @@ -30,13 +30,6 @@ class Paginator: spec: Mapping[str, Any] -@dataclass(frozen=True) -class Auth: - """Which of the connection's settings authenticate requests.""" - - spec: Mapping[str, Any] - - @dataclass(frozen=True) class Cells: """How each row's list of cells becomes columns; `paginate` and `fan_out` take it as `cells=`.""" @@ -146,57 +139,6 @@ def _location(location: str) -> str: return location -def bearer() -> Auth: - """`Authorization: Bearer `, the connection's `token`.""" - return Auth({"kind": "bearer"}) - - -def api_key(*, header: str | None = None, query: str | None = None) -> Auth: - """The connection's `token` in the header or the query parameter named.""" - if (header is None) == (query is None): - raise ValueError("sl.api_key takes exactly one of header= or query=") - return Auth({"kind": "api_key", "header": header, "query": query}) - - -def basic() -> Auth: - """HTTP basic auth with the connection's `username` and `password`.""" - return Auth({"kind": "basic"}) - - -def oauth2_client_credentials(token_url: str, *, scope: str | None = None, client_auth: str = "basic") -> Auth: - """`Authorization: Bearer `, the token `token_url` issues for the connection's `client_id` and `client_secret`. - - The token is fetched once per run, renewed before it expires and once after a 401. `client_auth` is `"basic"` - (HTTP basic auth at the token URL) or `"post"` (`client_id` and `client_secret` as form fields). - """ - return Auth(_oauth2("oauth2_client_credentials", token_url, scope, client_auth)) - - -def oauth2_refresh_token(token_url: str, *, scope: str | None = None, client_auth: str = "basic") -> Auth: - """`Authorization: Bearer `, the token `token_url` issues for the connection's `refresh_token`. - - The connection's `client_id` and `client_secret` go along when it has them. A refresh token the server rotates is - used for the rest of the run but not saved, so the next run starts from the connection's `refresh_token` again. - """ - return Auth(_oauth2("oauth2_refresh_token", token_url, scope, client_auth)) - - -def oauth2_password(token_url: str, *, scope: str | None = None, client_auth: str = "basic") -> Auth: - """`Authorization: Bearer `, the token `token_url` issues for the connection's `username` and `password`. - - The resource-owner password grant. The connection's `client_id` and `client_secret` go along when it has them. The - token is renewed before it expires and once after a 401: with the refresh token the server handed out, if it did, - and by asking with the password again otherwise or when that refresh token is refused. - """ - return Auth(_oauth2("oauth2_password", token_url, scope, client_auth)) - - -def _oauth2(kind: str, token_url: str, scope: str | None, client_auth: str) -> dict[str, Any]: - if client_auth not in ("basic", "post"): - raise ValueError(f'client_auth must be "basic" or "post", not {client_auth!r}') - return {"kind": kind, "token_url": token_url, "scope": scope, "client_auth": client_auth} - - def _param_text(value: Any) -> str: """A request parameter JSON has no form for, such as a `datetime` cursor value: ISO text for dates and times, else `str()`.""" isoformat = getattr(value, "isoformat", None) @@ -240,22 +182,18 @@ def __init__( self, source: Source, *, - auth: Auth | None = None, - rate_limit: str | None = None, - retries: int = DEFAULT_RETRIES, headers: Mapping[str, str] | None = None, connection: str | None = None, + **moved: Any, ) -> None: name = connection or source.settings.get("connection") if not name: raise ValueError(f"sl.http needs a connection: set connection= on source {source.name} or pass connection=") - self._client = { - "connection": name, - "auth": dict(auth.spec) if auth else {"kind": "none"}, - "rate_limit": rate_limit, - "retries": retries, - "headers": dict(headers or {}), - } + for key in moved: + if key in CONNECTION_SETTINGS: + raise TypeError(f"sl.http no longer takes {key}: set {key} on [connections.{name}] in sqlload.toml") + raise TypeError(f"sl.http got an unexpected keyword argument {key!r}") + self._client = {"connection": name, "headers": dict(headers or {})} def paginate( self, @@ -445,12 +383,11 @@ def _projection(fields: Fields | None, keep_record: str | None) -> dict[str, Any def http( source: Source, *, - auth: Auth | None = None, - rate_limit: str | None = None, - retries: int = DEFAULT_RETRIES, headers: Mapping[str, str] | None = None, connection: str | None = None, + **moved: Any, ) -> Http: - """Requests to `source`'s HTTP connection (or `connection`), with `auth`, a `rate_limit` such as `"10/s"` shared - by every job on the connection, and `retries` of transport errors, 429s and 5xx responses.""" - return Http(source, auth=auth, rate_limit=rate_limit, retries=retries, headers=headers, connection=connection) + """Requests to `source`'s HTTP connection (or `connection`), adding `headers` to the connection's. The connection in + sqlload.toml sets how requests authenticate (`auth`), the limits every request to it shares in the run (`rate_limit`, + `max_in_flight`) and its `retries`.""" + return Http(source, headers=headers, connection=connection, **moved) diff --git a/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sources/shop.py b/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sources/shop.py index d3788d282..bdd91d1ae 100644 --- a/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sources/shop.py +++ b/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sources/shop.py @@ -3,7 +3,7 @@ shop = sl.source( "shop", connection="shop_api", table_name="{table}", write_strategy="merge" ) -api = sl.http(shop, auth=sl.bearer(), retries=5) +api = sl.http(shop) @shop.table( diff --git a/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sqlload.toml b/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sqlload.toml index 2e9f8380e..e2103ea48 100644 --- a/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sqlload.toml +++ b/packages/sqlload/tests/docs_examples/from_dlt/rest_api/sqlload.toml @@ -5,6 +5,8 @@ default_target = "warehouse" kind = "http" base_url = "${ENV:SHOP_API_URL}" token = "${ENV:SHOP_TOKEN}" +auth = { kind = "bearer" } +retries = 5 [connections.duckdb] kind = "duckdb" diff --git a/packages/sqlload/tests/test_http_cells.py b/packages/sqlload/tests/test_http_cells.py index 577d5ecfe..37c1291fb 100644 --- a/packages/sqlload/tests/test_http_cells.py +++ b/packages/sqlload/tests/test_http_cells.py @@ -154,6 +154,8 @@ def table_project(tmp_path: Path, server: ThreadingHTTPServer, job: str, setting kind = "http" base_url = "http://{host}:{port}/v1" token = "${{ENV:SHOP_TOKEN}}" + auth = {{ kind = "bearer" }} + retries = 0 [targets.dev] connection = "wh" @@ -161,7 +163,7 @@ def table_project(tmp_path: Path, server: ThreadingHTTPServer, job: str, setting """) ) module = ( - "import sqlload as sl\n\nshop = sl.source('shop', write_strategy='replace', connection='shop')\napi = sl.http(shop, auth=sl.bearer(), retries=0)\n\n" + "import sqlload as sl\n\nshop = sl.source('shop', write_strategy='replace', connection='shop')\napi = sl.http(shop)\n\n" f"@shop.table(description='Rows.'{settings})\ndef rows(ctx):\n return {job}\n" ) (tmp_path / "sources" / "shop.py").write_text(module) diff --git a/packages/sqlload/tests/test_http_jobs.py b/packages/sqlload/tests/test_http_jobs.py index cf4f8942d..69dd735c3 100644 --- a/packages/sqlload/tests/test_http_jobs.py +++ b/packages/sqlload/tests/test_http_jobs.py @@ -383,8 +383,14 @@ def api(elsewhere: ThreadingHTTPServer) -> Iterator[ThreadingHTTPServer]: server.shutdown() -def make_project(tmp_path: Path, server: ThreadingHTTPServer, tables: str) -> Path: +BEARER = 'auth = { kind = "bearer" }\nretries = 3' +QUICK = 'auth = { kind = "bearer" }\nretries = 0' + + +def make_project(tmp_path: Path, server: ThreadingHTTPServer, tables: str, connection: str = BEARER) -> Path: + """A project whose `shop` connection also sets `connection` (where `TOKEN_URL` is the stub's token endpoint), with `api = sl.http(shop)`.""" host, port = server.server_address[:2] + connection = connection.replace("TOKEN_URL", f"http://{host}:{port}/oauth/token") (tmp_path / "sources").mkdir() (tmp_path / "sqlload.toml").write_text( textwrap.dedent(f"""\ @@ -403,13 +409,14 @@ def make_project(tmp_path: Path, server: ThreadingHTTPServer, tables: str) -> Pa refresh_token = "{REFRESH_TOKEN}" username = "loader" password = "${{ENV:SHOP_PASSWORD}}" + CONNECTION [targets.dev] connection = "wh" loader_schema = "RAW" - """) + """).replace("CONNECTION", connection) ) - module = f"import sqlload as sl\n\nTOKEN_URL = 'http://{host}:{port}/oauth/token'\nshop = sl.source('shop', write_strategy='replace', connection='shop')\napi = sl.http(shop, auth=sl.bearer(), retries=3)\n\n" + textwrap.dedent(tables) + module = f"import sqlload as sl\n\nTOKEN_URL = 'http://{host}:{port}/oauth/token'\nshop = sl.source('shop', write_strategy='replace', connection='shop')\napi = sl.http(shop)\n\n" + textwrap.dedent(tables) (tmp_path / "sources" / "shop.py").write_text(module) return tmp_path @@ -610,7 +617,7 @@ def orders(ctx): "HTTP 400", ), ( - "versioned = sl.http(shop, auth=sl.bearer(), headers={'X-Version': '1', 'X-Session-Id': 'abcdef12', 'X-Credential': 'abc123secret'})\n\n@shop.table(description='T.')\ndef t(ctx):\n return versioned.paginate('/echo', paginator=sl.single())\n", + "versioned = sl.http(shop, headers={'X-Version': '1', 'X-Session-Id': 'abcdef12', 'X-Credential': 'abc123secret'})\n\n@shop.table(description='T.')\ndef t(ctx):\n return versioned.paginate('/echo', paginator=sl.single())\n", "version 1 session *** credential ***", ), ( @@ -815,7 +822,7 @@ def test_given_fan_out_keys_in_a_post_body_when_loading_then_each_request_carrie def test_given_a_batch_that_fails_when_loading_then_the_error_names_its_keys(tmp_path: Path, api: ThreadingHTTPServer) -> None: job = "quick.fan_out('/lookup-flaky', method='POST', json_body={'ids': '{order_id}'}, batch=3, concurrency=1, over=range(1, 8), select='$.items')" - project = make_project(tmp_path, api, f"quick = sl.http(shop, auth=sl.bearer(), retries=0)\n\n@shop.table(description='Items.')\ndef items(ctx):\n return {job}\n") + project = make_project(tmp_path, api, f"quick = api\n\n@shop.table(description='Items.')\ndef items(ctx):\n return {job}\n", QUICK) done = sqlload(project, "load", "--json") assert done.returncode == 75, done.stdout + done.stderr (table,) = json.loads(done.stdout)["tables"] @@ -832,7 +839,7 @@ def test_given_a_body_placeholder_no_key_gives_when_loading_then_no_request_is_s FAIL_FAST_TABLES = """\ -quick = sl.http(shop, auth=sl.bearer(), retries=0) +quick = api @shop.table(description='Fails at once.') def a_fails(ctx): @@ -864,7 +871,7 @@ def d_later(ctx): def test_given_fail_fast_when_a_table_fails_while_another_loads_then_that_one_finishes_and_no_other_starts( tmp_path: Path, api: ThreadingHTTPServer, path: str, fail_fast: bool, expected_exit: int, expected_statuses: list[str] ) -> None: - project = make_project(tmp_path, api, FAIL_FAST_TABLES.format(path=path)) + project = make_project(tmp_path, api, FAIL_FAST_TABLES.format(path=path), QUICK) flags = ["--fail-fast"] if fail_fast else [] done = sqlload(project, "load", "--json", "--concurrency", "2", *flags) assert done.returncode == expected_exit, done.stdout + done.stderr @@ -946,8 +953,8 @@ def test_given_read_ahead_under_a_memory_budget_when_pages_are_larger_than_a_qua def test_given_read_ahead_and_a_rate_limit_when_loading_then_every_request_waits_its_turn(tmp_path: Path, api: ThreadingHTTPServer) -> None: - tables = "limited = sl.http(shop, auth=sl.bearer(), rate_limit='10/s')\n\n@shop.table(description='T.')\ndef t(ctx):\n return limited.paginate('/ahead', paginator=sl.offset(limit=1, read_ahead=4))\n" - project = make_project(tmp_path, api, tables) + tables = "@shop.table(description='T.')\ndef t(ctx):\n return api.paginate('/ahead', paginator=sl.offset(limit=1, read_ahead=4))\n" + project = make_project(tmp_path, api, tables, BEARER + '\nrate_limit = "10/s"') done = sqlload(project, "load", "--events", "jsonl") assert done.returncode == 0, done.stdout + done.stderr assert query(project, 'SELECT count(*) FROM "RAW".shop__t') == [(len(ORDERS),)] @@ -975,8 +982,8 @@ def test_given_offset_paging_when_the_api_rounds_the_offset_to_a_page_then_a_sho def test_given_an_api_key_in_the_query_when_loading_then_it_is_sent_only_to_the_connection(tmp_path: Path, api: ThreadingHTTPServer) -> None: - tables = "keyed = sl.http(shop, auth=sl.api_key(query='key'))\n\n@shop.table(description='T.')\ndef t(ctx):\n return keyed.paginate('/offset', paginator=sl.offset(limit=10))\n" - project = make_project(tmp_path, api, tables) + tables = "@shop.table(description='T.')\ndef t(ctx):\n return api.paginate('/offset', paginator=sl.offset(limit=10))\n" + project = make_project(tmp_path, api, tables, 'auth = { kind = "api_key", query = "key" }') done = sqlload(project, "load") assert done.returncode == 0, done.stdout + done.stderr assert [(path, query.get("key"), auth) for path, query, auth in api.log] == [("/v1/offset", TOKEN, None)] * 2 # type: ignore[attr-defined] @@ -993,8 +1000,8 @@ def test_given_an_api_key_in_the_query_when_loading_then_it_is_sent_only_to_the_ def test_given_a_response_pointing_at_another_host_when_loading_then_no_credential_or_header_goes_there( tmp_path: Path, api: ThreadingHTTPServer, elsewhere: ThreadingHTTPServer, path: str, paginator: str, expected_text: str ) -> None: - tables = f"keyed = sl.http(shop, auth=sl.api_key(header='X-Api-Key'), headers={{'X-Team': 'blue'}})\n\n@shop.table(description='T.')\ndef t(ctx):\n return keyed.paginate('{path}', paginator={paginator}, select='$.items')\n" - project = make_project(tmp_path, api, tables) + tables = f"keyed = sl.http(shop, headers={{'X-Team': 'blue'}})\n\n@shop.table(description='T.')\ndef t(ctx):\n return keyed.paginate('{path}', paginator={paginator}, select='$.items')\n" + project = make_project(tmp_path, api, tables, 'auth = { kind = "api_key", header = "X-Api-Key" }') done = sqlload(project, "load") assert expected_text in done.stdout + done.stderr, done.stdout + done.stderr sent = {name.lower() for headers in elsewhere.seen for name in headers} # type: ignore[attr-defined] @@ -1110,13 +1117,14 @@ def test_given_a_masked_table_when_rows_keep_their_record_then_the_load_is_refus assert api.log == [] # type: ignore[attr-defined] def oauth_project(tmp_path: Path, server: ThreadingHTTPServer, auth: str, job: str) -> Path: - tables = f"oauth = sl.http(shop, auth={auth})\n\n@shop.table(description='T.')\ndef t(ctx):\n return oauth.{job}\n" - return make_project(tmp_path, server, tables) + """A project whose connection sets `auth` (TOML, after `auth = `), with one table running `job`.""" + tables = f"@shop.table(description='T.')\ndef t(ctx):\n return api.{job}\n" + return make_project(tmp_path, server, tables, f"auth = {auth}") @pytest.mark.parametrize( ("auth", "expected_grant"), - [("sl.oauth2_client_credentials(TOKEN_URL, scope='orders:read')", "client_credentials"), ("sl.oauth2_password(TOKEN_URL, scope='orders:read')", "password")], + [('{ kind = "oauth2_client_credentials", token_url = "TOKEN_URL", scope = "orders:read" }', "client_credentials"), ('{ kind = "oauth2_password", token_url = "TOKEN_URL", scope = "orders:read" }', "password")], ids=["client-credentials", "password"], ) def test_given_short_lived_tokens_when_pages_outlast_one_then_it_is_renewed_before_it_expires( @@ -1135,9 +1143,9 @@ def test_given_short_lived_tokens_when_pages_outlast_one_then_it_is_renewed_befo @pytest.mark.parametrize( ("auth", "expected_grant"), [ - ("sl.oauth2_client_credentials(TOKEN_URL)", [("client_credentials", None)] * 3), - ("sl.oauth2_refresh_token(TOKEN_URL, client_auth='post')", [("refresh_token", REFRESH_TOKEN), ("refresh_token", "r3fresh-2"), ("refresh_token", "r3fresh-3")]), - ("sl.oauth2_password(TOKEN_URL)", [("password", None), ("refresh_token", "r3fresh-2"), ("refresh_token", "r3fresh-3")]), + ('{ kind = "oauth2_client_credentials", token_url = "TOKEN_URL" }', [("client_credentials", None)] * 3), + ('{ kind = "oauth2_refresh_token", token_url = "TOKEN_URL", client_auth = "post" }', [("refresh_token", REFRESH_TOKEN), ("refresh_token", "r3fresh-2"), ("refresh_token", "r3fresh-3")]), + ('{ kind = "oauth2_password", token_url = "TOKEN_URL" }', [("password", None), ("refresh_token", "r3fresh-2"), ("refresh_token", "r3fresh-3")]), ], ids=["client-credentials", "refresh-token-rotated-client-in-form", "password-then-its-refresh-token"], ) @@ -1160,22 +1168,22 @@ def test_given_a_token_revoked_mid_pagination_when_the_api_answers_401_then_a_ne ("auth", "job", "expected_text"), [ ( - "sl.oauth2_client_credentials(TOKEN_URL, client_auth='post')", + '{ kind = "oauth2_client_credentials", token_url = "TOKEN_URL", client_auth = "post" }', "paginate('/offset', paginator=sl.offset(limit=10))", "/oauth/token (OAuth2 token): HTTP 401: invalid_client: unknown client ['app', '***'] or refresh token; check the connection's OAuth2 credentials", ), ( - "sl.oauth2_client_credentials('http://auth.example.com/token')", + '{ kind = "oauth2_client_credentials", token_url = "http://auth.example.com/token" }', "paginate('/offset', paginator=sl.offset(limit=10))", "token_url http://auth.example.com/token must be https", ), ( - "sl.oauth2_client_credentials(TOKEN_URL)", + '{ kind = "oauth2_client_credentials", token_url = "TOKEN_URL" }', "paginate('/whoami', paginator=sl.single())", 'HTTP 400: {"message": "you are Bearer ***"}', ), ( - "sl.oauth2_client_credentials(TOKEN_URL)", + '{ kind = "oauth2_client_credentials", token_url = "TOKEN_URL" }', "paginate('/leak', paginator=sl.next_link('$.next'), select='$.items')", "/v1/notjson?access_token=***: the response is not JSON", ), @@ -1200,7 +1208,7 @@ def test_given_a_password_grant_without_a_refresh_token_or_with_a_refused_one_wh tmp_path: Path, api: ThreadingHTTPServer ) -> None: api.oauth = {"uses": 3, "stale_refresh": True} # type: ignore[attr-defined] - project = oauth_project(tmp_path, api, "sl.oauth2_password(TOKEN_URL, client_auth='post')", "paginate('/offset', paginator=sl.offset(limit=1))") + project = oauth_project(tmp_path, api, '{ kind = "oauth2_password", token_url = "TOKEN_URL", client_auth = "post" }', "paginate('/offset', paginator=sl.offset(limit=1))") done = sqlload(project, "load") assert done.returncode == 0, done.stdout + done.stderr assert query(project, 'SELECT order_id, status FROM "RAW".shop__t ORDER BY order_id') == ALL_ORDERS @@ -1215,11 +1223,11 @@ def test_given_a_rotating_refresh_token_when_a_page_read_ahead_is_renewing_as_th ) -> None: api.oauth = {"rotate": True, "slow_refresh": 1.0} # type: ignore[attr-defined] tables = ( - "oauth = sl.http(shop, auth=sl.oauth2_refresh_token(TOKEN_URL))\n\n" + "oauth = api\n\n" "@shop.table(description='T.')\ndef a_ahead(ctx):\n return oauth.paginate('/ahead_end', paginator=sl.offset(limit=1, read_ahead=2))\n\n" "@shop.table(description='T.')\ndef b_after(ctx):\n return oauth.paginate('/refuse_once', paginator=sl.single())\n" ) - project = make_project(tmp_path, api, tables) + project = make_project(tmp_path, api, tables, 'auth = { kind = "oauth2_refresh_token", token_url = "TOKEN_URL" }') done = sqlload(project, "load", "--concurrency", "1") assert done.returncode == 0, done.stdout + done.stderr for table in ("a_ahead", "b_after"): @@ -1232,7 +1240,7 @@ def test_given_a_token_endpoint_down_once_when_refreshing_after_a_password_grant tmp_path: Path, api: ThreadingHTTPServer ) -> None: api.oauth = {"uses": 3, "rotate": True, "refresh_503_once": True} # type: ignore[attr-defined] - project = oauth_project(tmp_path, api, "sl.oauth2_password(TOKEN_URL)", "paginate('/offset', paginator=sl.offset(limit=1))") + project = oauth_project(tmp_path, api, '{ kind = "oauth2_password", token_url = "TOKEN_URL" }', "paginate('/offset', paginator=sl.offset(limit=1))") done = sqlload(project, "load") assert done.returncode == 0, done.stdout + done.stderr assert query(project, 'SELECT order_id, status FROM "RAW".shop__t ORDER BY order_id') == ALL_ORDERS @@ -1244,14 +1252,14 @@ def test_given_a_token_endpoint_down_when_refreshing_without_retries_then_the_ru tmp_path: Path, api: ThreadingHTTPServer ) -> None: api.oauth = {"uses": 3, "rotate": True, "refresh_503_once": True} # type: ignore[attr-defined] - project = oauth_project(tmp_path, api, "sl.oauth2_password(TOKEN_URL), retries=0", "paginate('/offset', paginator=sl.offset(limit=1))") + project = oauth_project(tmp_path, api, '{ kind = "oauth2_password", token_url = "TOKEN_URL" }\nretries = 0', "paginate('/offset', paginator=sl.offset(limit=1))") done = sqlload(project, "load") assert done.returncode == 75, done.stdout + done.stderr assert [(request["grant_type"], request.get("refresh_token")) for request in api.token_requests] == [("password", None), ("refresh_token", "r3fresh-2")] # type: ignore[attr-defined] def test_given_a_wrong_password_when_loading_with_json_then_it_fails_for_good_and_hides_the_password(tmp_path: Path, api: ThreadingHTTPServer) -> None: - project = oauth_project(tmp_path, api, "sl.oauth2_password(TOKEN_URL)", "paginate('/offset', paginator=sl.offset(limit=10))") + project = oauth_project(tmp_path, api, '{ kind = "oauth2_password", token_url = "TOKEN_URL" }', "paginate('/offset', paginator=sl.offset(limit=10))") (project / "sqlload.toml").write_text((project / "sqlload.toml").read_text().replace("${ENV:SHOP_PASSWORD}", "wr0ng+p\u00e4ss\\\\word")) done = sqlload(project, "load", "--json") assert done.returncode == 1, done.stdout + done.stderr @@ -1263,7 +1271,7 @@ def test_given_a_wrong_password_when_loading_with_json_then_it_fails_for_good_an def test_given_401_listed_as_skip_when_the_api_answers_it_then_the_request_is_skipped_without_a_new_token(tmp_path: Path, api: ThreadingHTTPServer) -> None: - project = oauth_project(tmp_path, api, "sl.oauth2_client_credentials(TOKEN_URL)", "fan_out('/denied?k={k}', over=range(5), concurrency=1, paginator=sl.single(), on_status={401: 'skip'})") + project = oauth_project(tmp_path, api, '{ kind = "oauth2_client_credentials", token_url = "TOKEN_URL" }', "fan_out('/denied?k={k}', over=range(5), concurrency=1, paginator=sl.single(), on_status={401: 'skip'})") done = sqlload(project, "load") assert done.returncode == 0, done.stdout + done.stderr assert len([path for path, _, _ in api.log if path == "/v1/denied"]) == 5 # type: ignore[attr-defined] @@ -1273,7 +1281,7 @@ def test_given_401_listed_as_skip_when_the_api_answers_it_then_the_request_is_sk def test_given_a_401_from_another_origin_when_loading_then_no_token_is_sent_there_or_renewed_for_it( tmp_path: Path, api: ThreadingHTTPServer, elsewhere: ThreadingHTTPServer ) -> None: - project = oauth_project(tmp_path, api, "sl.oauth2_client_credentials(TOKEN_URL)", "paginate('/away401', paginator=sl.next_link('$.next'), select='$.items')") + project = oauth_project(tmp_path, api, '{ kind = "oauth2_client_credentials", token_url = "TOKEN_URL" }', "paginate('/away401', paginator=sl.next_link('$.next'), select='$.items')") done = sqlload(project, "load") assert done.returncode == 1 assert "/denied: HTTP 401" in done.stdout + done.stderr, done.stdout + done.stderr @@ -1345,7 +1353,7 @@ def test_given_a_post_answered_with_a_redirect_or_link_when_loading_then_its_bod def test_given_a_connection_content_type_when_posting_then_it_is_the_only_one_sent(tmp_path: Path, api: ThreadingHTTPServer) -> None: tables = ( - "typed = sl.http(shop, auth=sl.bearer(), headers={'Content-Type': 'application/json; charset=utf-8'})\n\n" + "typed = sl.http(shop, headers={'Content-Type': 'application/json; charset=utf-8'})\n\n" "@shop.table(description='Orders.')\ndef orders(ctx):\n" " return typed.paginate('/search-cursor', method='POST', paginator=sl.single(), select='$.items')\n" ) @@ -1615,7 +1623,9 @@ def test_given_a_failing_level_when_loading_then_the_error_names_the_keys_that_l ) -> None: project = tree_project(tmp_path, api, notes, on_status=on_status) shop_py = project / "sources" / "shop.py" - shop_py.write_text(shop_py.read_text().replace("\n\n@shop.table", "\nonce = sl.http(shop, auth=sl.bearer(), retries=0)\n\n@shop.table", 1)) + shop_py.write_text(shop_py.read_text().replace("\n\n@shop.table", "\nonce = api\n\n@shop.table", 1)) + toml = project / "sqlload.toml" + toml.write_text(toml.read_text().replace("retries = 3", "retries = 0")) done = sqlload(project, "load") output = done.stdout + done.stderr assert done.returncode == expected_code, output @@ -1636,10 +1646,10 @@ def test_given_a_cursor_on_the_top_level_when_loading_twice_then_the_rerun_reque ONE_LEVEL_JOB = ( - '{"kind": "fan_out", "client": {"connection": "shop", "auth": {"kind": "bearer"}, "rate_limit": null, "retries": 3, "headers": {}}, ' + '{"kind": "fan_out", "client": {"connection": "shop", "headers": {}}, ' '"path": "/orders/{order_id}/items", "params": {}, "method": "GET", "json_body": null, "paginator": {"kind": "single"}, "select": "$.items", ' '"on_status": {"404": "skip"}, "fields": null, "keep_record": null, "over": {"kind": "paginate", "client": {"connection": "shop", ' - '"auth": {"kind": "bearer"}, "rate_limit": null, "retries": 3, "headers": {}}, "path": "/orders", "params": {}, "method": "GET", ' + '"headers": {}}, "path": "/orders", "params": {}, "method": "GET", ' '"json_body": null, "paginator": {"kind": "page_number", "param": "p", "start": 1, "location": "query"}, "select": "$.items[*].order_id", ' '"on_status": {}, "fields": null, "keep_record": null}, "concurrency": 3}' ) @@ -1647,7 +1657,7 @@ def test_given_a_cursor_on_the_top_level_when_loading_twice_then_the_rerun_reque def test_given_a_one_level_fan_out_when_defined_then_its_job_is_unchanged_by_multi_level_support() -> None: code = ( - "import sqlload as sl\nshop = sl.source('shop', write_strategy='replace', connection='shop')\napi = sl.http(shop, auth=sl.bearer(), retries=3)\n" + "import sqlload as sl\nshop = sl.source('shop', write_strategy='replace', connection='shop')\napi = sl.http(shop)\n" "orders = api.paginate('/orders', paginator=sl.page_number('p'), select='$.items[*].order_id')\n" "print(api.fan_out('/orders/{order_id}/items', over=orders, concurrency=3, select='$.items', on_status={404: 'skip'}).__sqlload_http_job__())" ) diff --git a/packages/sqlload/tests/test_http_limits.py b/packages/sqlload/tests/test_http_limits.py new file mode 100644 index 000000000..cba3f4b63 --- /dev/null +++ b/packages/sqlload/tests/test_http_limits.py @@ -0,0 +1,188 @@ +"""The limits an `http` connection sets for every request to its API in one run: `rate_limit` paces request starts and +`max_in_flight` caps open requests, both shared by every table, page, `read_ahead` and retry on the connection. The stub +API answers slowly and records when each request arrived and how many were open at once.""" + +import json +import os +import subprocess +import sys +import textwrap +import threading +import time +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +import duckdb +import pytest + +import sqlload as sl + +PAGES = 16 +TOKEN = "l1mits-token" + + +class Api(BaseHTTPRequestHandler): + """`/?offset&limit`: `PAGES` rows of one row a page, each answer after `self.server.delay` seconds; every request's + arrival time goes in `self.server.arrivals`, and the most requests open at once (arrived and not yet answered) in + `self.server.most_open`. With `self.server.throttle`, every third request is answered 429 with `Retry-After: 0`.""" + + def log_message(self, *args: Any) -> None: + pass + + def do_GET(self) -> None: + server: Any = self.server + url = urlparse(self.path) + query = {key: values[0] for key, values in parse_qs(url.query).items()} + with server.lock: + server.arrivals.append(time.monotonic()) + server.open += 1 + server.most_open = max(server.most_open, server.open) + server.auth.add(self.headers.get("Authorization")) + throttled = server.throttle and len(server.arrivals) % 3 == 0 + time.sleep(server.delay) + # The request stops counting as open before any of its answer is sent: once the client has the answer it may send + # the next request, which could arrive before this thread ran again to count this one closed. + with server.lock: + server.open -= 1 + if throttled: + self.send_response(429) + self.send_header("Retry-After", "0") + self.send_header("Content-Length", "0") + self.end_headers() + return + offset, limit = int(query.get("offset", 0)), int(query.get("limit", 1)) + rows = [{"id": n, "table": url.path} for n in range(offset, min(offset + limit, PAGES))] + data = json.dumps(rows).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +@pytest.fixture +def api() -> Iterator[ThreadingHTTPServer]: + server = ThreadingHTTPServer(("127.0.0.1", 0), Api) + server.daemon_threads = True + server.lock = threading.Lock() # type: ignore[attr-defined] + server.arrivals, server.auth = [], set() # type: ignore[attr-defined] + server.open, server.most_open, server.delay, server.throttle = 0, 0, 0.0, False # type: ignore[attr-defined] + threading.Thread(target=server.serve_forever, daemon=True).start() + yield server + server.shutdown() + + +def make_project(tmp_path: Path, server: ThreadingHTTPServer, limits: str, tables: int) -> Path: + """A project whose `shop` connection sets `limits`, with `tables` tables of one source paging `/t` at the same time.""" + host, port = server.server_address[:2] + (tmp_path / "sources").mkdir() + (tmp_path / "sqlload.toml").write_text( + textwrap.dedent(f"""\ + python_paths = ["sources"] + + [connections.wh] + kind = "duckdb" + database = "wh.duckdb" + + [connections.shop] + kind = "http" + base_url = "http://{host}:{port}/v1" + token = "${{ENV:SHOP_TOKEN}}" + auth = {{ kind = "bearer" }} + {limits} + + [targets.dev] + connection = "wh" + loader_schema = "RAW" + """) + ) + module = "import sqlload as sl\n\nshop = sl.source('shop', write_strategy='replace', connection='shop', parallel_tables=True)\napi = sl.http(shop)\n" + for n in range(tables): + module += f"\n@shop.table(name='t{n}', description='T.')\ndef t{n}(ctx):\n return api.paginate('/t{n}', paginator=sl.offset(limit=1, read_ahead=20))\n" + (tmp_path / "sources" / "shop.py").write_text(module) + return tmp_path + + +def sqlload(project: Path, *args: str) -> subprocess.CompletedProcess[str]: + home = project / "home" + home.mkdir(exist_ok=True) + env = {**os.environ, "HOME": str(home), "XDG_CONFIG_HOME": str(home / ".config"), "SHOP_TOKEN": TOKEN} + env.pop("NO_COLOR", None) + script = "from sqlload._cli import main; main()" + return subprocess.run([sys.executable, "-c", script, *args], cwd=project, env=env, capture_output=True, text=True, timeout=120) + + +def loaded(project: Path, tables: int) -> list[int]: + with duckdb.connect(str(project / "wh.duckdb"), read_only=True) as connection: + return [connection.execute(f'SELECT count(*) FROM "RAW".shop__t{n}').fetchall()[0][0] for n in range(tables)] + + +def test_given_two_tables_on_one_connection_when_they_load_at_once_then_together_they_stay_under_its_rate_limit( + tmp_path: Path, api: ThreadingHTTPServer +) -> None: + project = make_project(tmp_path, api, 'rate_limit = "20/s"', tables=2) + done = sqlload(project, "load", "--concurrency", "4", "--events", "jsonl") + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(project, 2) == [PAGES, PAGES] + arrivals: list[float] = sorted(api.arrivals) # type: ignore[attr-defined] + # One limit of 20 a second for both tables: the requests' starts span at least (n - 1) / 20 seconds. Two limits, one per + # table, would let them through in about half that. A slow machine only stretches the span, so the bound holds anywhere. + assert arrivals[-1] - arrivals[0] >= (len(arrivals) - 1) / 20 * 0.95, (len(arrivals), arrivals[-1] - arrivals[0]) + assert api.auth == {f"Bearer {TOKEN}"} # type: ignore[attr-defined] + events = [json.loads(line) for line in done.stderr.splitlines() if line.startswith("{")] + waits = [event for event in events if event["event"] == "rate_limited"] + assert waits and {(event["connection"], event["rate"]) for event in waits} == {("shop", "20/s")}, done.stderr + + +def test_given_max_in_flight_when_tables_read_far_ahead_at_once_then_the_api_never_sees_more_open_requests( + tmp_path: Path, api: ThreadingHTTPServer +) -> None: + api.delay = 0.1 # type: ignore[attr-defined] + project = make_project(tmp_path, api, "max_in_flight = 3", tables=4) + done = sqlload(project, "load", "--concurrency", "4") + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(project, 4) == [PAGES] * 4 + # Four tables reading 20 pages ahead could hold 80 requests open; the cap holds them at 3, and more than one was open at once. + assert 2 <= api.most_open <= 3, api.most_open # type: ignore[attr-defined] + + +def test_given_429s_when_requests_are_retried_then_each_retry_counts_against_both_limits(tmp_path: Path, api: ThreadingHTTPServer) -> None: + api.delay, api.throttle = 0.05, True # type: ignore[attr-defined] + project = make_project(tmp_path, api, 'rate_limit = "20/s"\nmax_in_flight = 2', tables=2) + done = sqlload(project, "load", "--concurrency", "4") + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(project, 2) == [PAGES, PAGES] + arrivals: list[float] = sorted(api.arrivals) # type: ignore[attr-defined] + # A third of the requests were answered 429 and sent again: every attempt, retries included, took a turn and a permit. + assert len(arrivals) >= 2 * PAGES * 3 // 2, len(arrivals) + assert arrivals[-1] - arrivals[0] >= (len(arrivals) - 1) / 20 * 0.95, (len(arrivals), arrivals[-1] - arrivals[0]) + assert api.most_open <= 2, api.most_open # type: ignore[attr-defined] + + +def test_given_an_http_connection_with_limits_when_planning_then_plan_shows_them_and_no_secret(tmp_path: Path, api: ThreadingHTTPServer) -> None: + project = make_project(tmp_path, api, 'rate_limit = "600/m"\nmax_in_flight = 4\nretries = 2', tables=1) + done = sqlload(project, "plan", "-v") + assert done.returncode == 0, done.stdout + done.stderr + assert "http shop rate_limit 600/m, max_in_flight 4, retries 2, auth bearer" in done.stdout, done.stdout + assert TOKEN not in done.stdout + done.stderr + + +@pytest.mark.parametrize( + ("argument", "expected_text"), + [ + ({"rate_limit": "10/s"}, "sl.http no longer takes rate_limit: set rate_limit on [connections.shop] in sqlload.toml"), + ({"retries": 3}, "sl.http no longer takes retries: set retries on [connections.shop] in sqlload.toml"), + ({"auth": "bearer"}, "sl.http no longer takes auth: set auth on [connections.shop] in sqlload.toml"), + ], + ids=["rate_limit", "retries", "auth"], +) +def test_given_a_limit_passed_to_sl_http_when_building_the_client_then_it_names_the_connection_key_to_set( + argument: dict[str, Any], expected_text: str +) -> None: + shop = sl.source("shop", write_strategy="replace", connection="shop") + with pytest.raises(TypeError) as raised: + sl.http(shop, **argument) + assert str(raised.value) == expected_text diff --git a/website/public/index.html b/website/public/index.html index 75d491c87..bc64ff272 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -251,7 +251,7 @@

APIs without the plumbing

import sqlload as sl
 
 shop = sl.source("shop", connection="shop")
-api = sl.http(shop, auth=sl.bearer(), rate_limit="10/s")
+api = sl.http(shop)  # auth, rate_limit, max_in_flight: [connections.shop]
 
 @shop.table(unique_key="order_id")
 def orders(ctx):
diff --git a/website/src/content/docs/docs/cli/debug.mdx b/website/src/content/docs/docs/cli/debug.mdx
index b381c2c00..f71f4fce6 100644
--- a/website/src/content/docs/docs/cli/debug.mdx
+++ b/website/src/content/docs/docs/cli/debug.mdx
@@ -27,6 +27,14 @@ $ sqlload debug
 fail  source app  cannot connect to Postgres at db.example.com:5432/postgres: error performing TLS handshake: invalid peer certificate: UnknownIssuer
 ```
 
+An `http` connection's line says which of its settings the credentials come from, and for OAuth2
+the token URL. One that lacks a credential its `auth` needs prints `fail`:
+
+```console
+ok    connection shop  auth bearer: token from the connection
+fail  connection erp  auth oauth2_client_credentials needs client_secret on the connection
+```
+
 | Option | Meaning |
 |--------|---------|
 | `--no-connection` | Only validate the configuration; connect to nothing. |
diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx
index 3d8688bfa..391f935bd 100644
--- a/website/src/content/docs/docs/cli/plan.mdx
+++ b/website/src/content/docs/docs/cli/plan.mdx
@@ -105,6 +105,14 @@ The start is `≥` when the bound row is read again and `>` when it isn't, follo
 comes from: `(max)`, the warehouse maximum, `(max − 5s)` with a `lookback`, `--start-cursor`, or
 `from state "…" (run 20d2b872)` for a logged state.
 
+A project with an [`http` connection](/docs/configuration/#http) starts with a line for each,
+showing the limits every request to it shares in the run, its retries and its auth scheme, never its
+credentials:
+
+```console
+http  shop  rate_limit 10/s, max_in_flight 4, retries 5, auth bearer
+```
+
 A child line under a table means it does something unlike the defaults, or needs a look:
 
 | Line | Means |
diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx
index 62104e77a..eff9922ff 100644
--- a/website/src/content/docs/docs/configuration.mdx
+++ b/website/src/content/docs/docs/configuration.mdx
@@ -210,16 +210,53 @@ See [ODBC](/docs/sources/odbc/#connection).
 ### `http`
 
 For [Python HTTP sources](/docs/sources/python/#http-apis); its settings reach table functions as
-`ctx.connection`.
+`ctx.connection`. The connection holds everything about the API that every request shares: how
+requests authenticate, the limits they stay under together, and how often a failed one is retried.
 
-| Key | Meaning |
-|-----|---------|
-| `base_url` | The API's base URL. |
-| `token` | For `sl.bearer()` and `sl.api_key(...)`. |
-| `username`, `password` | For `sl.basic()`, and for `sl.oauth2_password(...)`. |
-| `client_id`, `client_secret` | For `sl.oauth2_client_credentials(...)`, and for `sl.oauth2_refresh_token(...)` or `sl.oauth2_password(...)` when the API wants the client too. |
-| `refresh_token` | For `sl.oauth2_refresh_token(...)`. |
-| `headers` | Headers sent with every request. |
+| Key | Default | Meaning |
+|-----|---------|---------|
+| `base_url` | required | The API's base URL. |
+| `auth` | none | How requests authenticate, a table such as `{ kind = "bearer" }`; see the table below. Without it, requests carry no credential. |
+| `token` | none | For `auth` `bearer` and `api_key`. Use `${ENV:NAME}`. |
+| `username`, `password` | none | For `auth` `basic` and `oauth2_password`. |
+| `client_id`, `client_secret` | none | For `auth` `oauth2_client_credentials`, and for `oauth2_refresh_token` or `oauth2_password` when the API wants the client too. |
+| `refresh_token` | none | For `auth` `oauth2_refresh_token`. |
+| `headers` | none | Headers sent with every request. |
+| `rate_limit` | none | Requests started per second, minute or hour: `"10/s"`, `"600/m"` or `"1000/h"`. |
+| `max_in_flight` | none | The most requests open at once, from 1 to 10,000. |
+| `retries` | 5 | Retries of a request that failed with a transport error, 429 or a 5xx, at most 100. |
+
+`rate_limit` and `max_in_flight` hold for every request to the connection in one `sqlload load`:
+every table, page, [`read_ahead`](/docs/sources/python/#reading-pages-ahead) request, fan-out key
+and retry. A retried request takes a turn and a place again. Separate processes don't share them,
+so load one API's tables in one `sqlload load`; in an orchestrator, that means one task per API, not
+one per endpoint.
+
+| `auth` | Sends |
+|--------|-------|
+| `{ kind = "bearer" }` | `Authorization: Bearer `. |
+| `{ kind = "api_key", header = "X-Api-Key" }` | `token` in that header. |
+| `{ kind = "api_key", query = "key" }` | `token` in that query parameter. |
+| `{ kind = "basic" }` | Basic auth with `username` and `password`. |
+| `{ kind = "oauth2_client_credentials", token_url = "...", scope = "...", client_auth = "basic" }` | `Authorization: Bearer `, the token `token_url` issues for `client_id` and `client_secret`. `scope` is optional; `client_auth` is `basic` (default) or `post`. See [OAuth2](/docs/sources/python/#oauth2). |
+| `{ kind = "oauth2_refresh_token", token_url = "..." }` | The same, with the token issued for `refresh_token`. Takes `scope` and `client_auth` too. |
+| `{ kind = "oauth2_password", token_url = "..." }` | The same, with the token issued for `username` and `password` (the resource-owner password grant). Takes `scope` and `client_auth` too. |
+
+```toml
+[connections.shop]
+kind = "http"
+base_url = "https://api.example.com/v1"
+token = "${ENV:SHOP_TOKEN}"
+auth = { kind = "bearer" }
+rate_limit = "10/s"
+max_in_flight = 4
+```
+
+`plan` shows each `http` connection's settings, never its credentials:
+
+```text
+http  shop  rate_limit 10/s, max_in_flight 4, retries 5, auth bearer
+```
 
 ## Targets
 
diff --git a/website/src/content/docs/docs/guides/from-dlt.mdx b/website/src/content/docs/docs/guides/from-dlt.mdx
index f34a54413..fcfb44d2a 100644
--- a/website/src/content/docs/docs/guides/from-dlt.mdx
+++ b/website/src/content/docs/docs/guides/from-dlt.mdx
@@ -100,12 +100,12 @@ holds now. `payments` reads every order's payments again and merges them on thei
 
 How `rest_api` settings map:
 
-- `client.base_url` and the credentials go in the `http` connection; `auth` is an argument of
-  `sl.http`. `bearer` is `sl.bearer()`, `api_key` is `sl.api_key(header=...)` or
-  `sl.api_key(query=...)`, `http_basic` is `sl.basic()`, and `oauth2_client_credentials` is
-  `sl.oauth2_client_credentials(token_url)`. The OAuth2 password grant, which dlt's `rest_api` has
-  no auth type for ([config_setup.py](https://github.com/dlt-hub/dlt/blob/90c0cea1f7b59795de91824be466c99efca0f574/dlt/sources/rest_api/config_setup.py#L89-L94)), is
-  `sl.oauth2_password(token_url)`; see [OAuth2](/docs/sources/python/#oauth2).
+- `client.base_url`, the credentials and `auth` go in the [`http` connection](/docs/configuration/#http).
+  `bearer` is `auth = { kind = "bearer" }`, `api_key` is `{ kind = "api_key", header = "..." }` or
+  `{ kind = "api_key", query = "..." }`, `http_basic` is `{ kind = "basic" }`, and
+  `oauth2_client_credentials` is `{ kind = "oauth2_client_credentials", token_url = "..." }`. The
+  OAuth2 password grant, which dlt's `rest_api` has no auth type for ([config_setup.py](https://github.com/dlt-hub/dlt/blob/90c0cea1f7b59795de91824be466c99efca0f574/dlt/sources/rest_api/config_setup.py#L89-L94)), is
+  `{ kind = "oauth2_password", token_url = "..." }`; see [OAuth2](/docs/sources/python/#oauth2).
 - Paginators: `json_link` is `sl.next_link("$.next")`, `header_link` is `sl.link_header()`,
   `offset` is `sl.offset(limit=...)`, `page_number` is `sl.page_number()`, `cursor` is
   `sl.cursor(pointer, param)`, and `single_page` is `sl.single()`. dlt's `auto` guesses one
@@ -127,7 +127,9 @@ How `rest_api` settings map:
   `transform=` on the job: a function called once per page with the page's rows; see
   [Transforming each page](/docs/sources/python/#transforming-each-page).
 - `response_actions` that ignore a status are `on_status={404: "skip"}`. 429, 5xx and dropped
-  connections are retried, honouring `Retry-After`; `retries=` on `sl.http` sets how many times.
+  connections are retried, honouring `Retry-After`; `retries` on the connection sets how many times.
+  A limit on the API's request rate or open requests is `rate_limit` or `max_in_flight` on the
+  connection, shared by every resource loading in the run.
 - An API that returns each row as a list of cells, such as a spreadsheet's or an EAM grid's, is
   `cells=sl.cells(..., name=...)`, whether each cell names its own column or the response lists
   them; see [Rows of cells](/docs/sources/python/#rows-of-cells).
diff --git a/website/src/content/docs/docs/guides/orchestrators.mdx b/website/src/content/docs/docs/guides/orchestrators.mdx
index 7e7dca437..e449879fc 100644
--- a/website/src/content/docs/docs/guides/orchestrators.mdx
+++ b/website/src/content/docs/docs/guides/orchestrators.mdx
@@ -9,6 +9,11 @@ each event to the orchestrator's logger at its level, and records a row per tabl
 fails logs an error, and the run fails when `sqlload load` exits with a non-zero code. Run every
 recipe in the project folder, with `sqlload` on `PATH`.
 
+An [`http` connection's](/docs/configuration/#http) `rate_limit` and `max_in_flight` hold within
+one `sqlload load`; separate processes don't share them. Run one task per API, loading all of its
+tables, not one task per endpoint. The recipes log each `rate_limited` event, which names the
+connection and its `rate_limit`, so a run that waits on its limit shows it.
+
 ## Prefect
 
 The flow logs through `get_run_logger()` and saves a table artifact, `sqlload-load`, with one row
diff --git a/website/src/content/docs/docs/guides/python.mdx b/website/src/content/docs/docs/guides/python.mdx
index abb626961..8024e592b 100644
--- a/website/src/content/docs/docs/guides/python.mdx
+++ b/website/src/content/docs/docs/guides/python.mdx
@@ -28,7 +28,7 @@ Rust, so there is no per-row Python cost.
 import sqlload as sl
 
 shop = sl.source("shop", connection="shop", write_strategy="merge")
-api = sl.http(shop, auth=sl.bearer(), rate_limit="10/s", retries=5)
+api = sl.http(shop)
 
 
 @shop.table(description="Orders, one row per order.", unique_key="order_id")
@@ -127,13 +127,21 @@ the base URL and credentials:
 kind = "http"
 base_url = "https://api.example.com"
 token = "${ENV:SHOP_TOKEN}"
+auth = { kind = "bearer" }
+rate_limit = "10/s"
+max_in_flight = 4
 ```
 
+The connection holds what every request to the API shares: `auth`, `rate_limit`, `max_in_flight`
+and `retries` (5 by default). The limits hold across every table in the run, so tables reading the
+same API at once stay under them together. Load one API's tables in one `sqlload load`; separate
+processes don't share the limits. See the [`http` connection](/docs/configuration/#http).
+
 ```python
 import sqlload as sl
 
 shop = sl.source("shop", connection="shop", write_strategy="merge")
-api = sl.http(shop, auth=sl.bearer(), rate_limit="10/s", retries=5)
+api = sl.http(shop)
 
 
 @shop.table(description="Orders, one row per order.", unique_key="order_id")
@@ -209,17 +217,13 @@ kind = "http"
 base_url = "https://api.example.com/v2"
 client_id = "loader"
 client_secret = "${ENV:ERP_CLIENT_SECRET}"
+auth = { kind = "oauth2_client_credentials", token_url = "https://auth.example.com/oauth/token", scope = "accounts:read" }
 ```
 
 ```python
 erp = sl.source("erp", connection="erp", write_strategy="replace")
-api = sl.http(
-    erp,
-    auth=sl.oauth2_client_credentials(
-        "https://auth.example.com/oauth/token", scope="accounts:read"
-    ),
-)
+api = sl.http(erp)
 ```
 
-SQLLoad fetches the token and renews it before it expires. For `sl.oauth2_refresh_token`, how
+SQLLoad fetches the token and renews it before it expires. For `oauth2_refresh_token`, how
 tokens are renewed and where secrets go, see [OAuth2](/docs/sources/python/#oauth2).
diff --git a/website/src/content/docs/docs/sources/python.mdx b/website/src/content/docs/docs/sources/python.mdx
index 7ab51068e..f69e29ff0 100644
--- a/website/src/content/docs/docs/sources/python.mdx
+++ b/website/src/content/docs/docs/sources/python.mdx
@@ -341,7 +341,7 @@ thread-safe. Set `parallel_tables=True` on the source when its client is.
 
 | Function | Meaning |
 |----------|---------|
-| `sl.http(source, *, auth=None, rate_limit=None, retries=5, headers=None, connection=None)` | Requests to the source's `http` connection. `rate_limit` (`"10/s"`, `"600/m"`, `"1000/h"`) is shared by every job on the connection with the same limit. |
+| `sl.http(source, *, headers=None, connection=None)` | Requests to the source's `http` connection, or to `connection`, adding `headers` to the connection's. How requests authenticate, their `rate_limit` and `max_in_flight`, and their `retries` are set on the [connection](/docs/configuration/#http); passing `auth`, `rate_limit`, `max_in_flight` or `retries` to `sl.http` fails, naming the connection key to set. |
 | `api.paginate(path, *, paginator, params=None, select=None, on_status=None, fields=None, keep_record=None, method="GET", json_body=None, key=None, cells=None, select_missing="fail", error_path=None, transform=None)` | Every page of `path`. `select` is where the rows are: `$`, or a dotted path such as `$.data.items`. A response without the `select` path fails the load, naming what the response holds; `select_missing="empty"` reads it as no rows instead. See [A missing select path](#a-missing-select-path). `error_path` is where the API reports an error in a successful response; see [Errors in a successful response](#errors-in-a-successful-response). `fields` and `keep_record` pick and rename each row's values; see [Fields](#fields). For `method` and `json_body`, see [POST with a JSON body](#post-with-a-json-body). `cells=` turns each row's list of cells into columns; see [Rows of cells](#rows-of-cells). `transform` reshapes each page's rows in Python before they load; see [Transforming each page](#transforming-each-page). |
 | `api.fan_out(path, *, over, concurrency=8, paginator=None, params=None, select=None, on_status=None, fields=None, keep_record=None, method="GET", json_body=None, key=None, cells=None, batch=None, select_missing="fail", error_path=None, transform=None)` | `path` once per key of `over`, up to `concurrency` at a time, while keys still arrive. `select_missing` and `error_path` are as for `paginate`. `over` is a paginate whose `select` picks keys (`$.items[*].id` fills `{id}`), or a list: a value fills the path's one placeholder, and a dict fills placeholders by name. `over` can also be another `fan_out`; see [Multi-level fan-out](#multi-level-fan-out). `method` and `json_body` are as for `paginate`. Keys also fill `{placeholders}` in `json_body`, and `batch` sends several keys per request; see [Keys in a POST body](#keys-in-a-post-body). `transform` is as for `paginate`, called once per page of each key. |
 
@@ -360,14 +360,14 @@ thread-safe. Set `parallel_tables=True` on the source when its client is.
 | `sl.listed(path, *, key="columnId", id_key="id", name_key="title", by_position=False)` | The columns the response lists at `path`, such as `"$.columns"`. Each cell's `key` matches a listed column's `id_key`, and `name_key` names it; `by_position=True` matches cells to columns by position. |
 | `sl.header_row()` | The first row's cells name the columns, by position, and that row doesn't load. |
 
-| Auth | Sends |
-|------|-------|
-| `sl.bearer()` | `Authorization: Bearer `, the connection's `token`. |
-| `sl.api_key(header=...)` or `sl.api_key(query=...)` | The `token` in that header or query parameter. |
-| `sl.basic()` | Basic auth with the connection's `username` and `password`. |
-| `sl.oauth2_client_credentials(token_url, *, scope=None, client_auth="basic")` | `Authorization: Bearer `, with the token `token_url` issues for the connection's `client_id` and `client_secret`. See [OAuth2](#oauth2). |
-| `sl.oauth2_refresh_token(token_url, *, scope=None, client_auth="basic")` | The same, with the token issued for the connection's `refresh_token`. The connection's `client_id` and `client_secret` go along when it has them. |
-| `sl.oauth2_password(token_url, *, scope=None, client_auth="basic")` | The same, with the token issued for the connection's `username` and `password` (the resource-owner password grant). The connection's `client_id` and `client_secret` go along when it has them. |
+The connection's `auth` says how requests authenticate: `bearer`, `api_key` in a header or query
+parameter, `basic`, or one of three OAuth2 grants. See the [`http` connection](/docs/configuration/#http)
+for each and the credentials it uses.
+
+- Every request to the connection in a run shares its `rate_limit` and `max_in_flight`, whichever
+  table, page, fan-out key or retry it belongs to, so two tables reading one API at once stay under
+  one limit together. Load one API's tables in one `sqlload load`: separate processes don't share
+  the limits.
 
 - Every `paginate` names its paginator; nothing is guessed.
 - Credentials go only to the connection's own scheme, host and port, never to a next-page link
@@ -392,7 +392,7 @@ api.paginate("/invoices", paginator=sl.offset(limit=50, read_ahead=20))
 
 The first page is read alone, so a short first page is checked as without `read_ahead`. After a full first page,
 the pages after it are requested two at a time, then four, eight and so on up to `N`, the open requests doubling
-each time the pages of the last round have loaded. Each goes through the shared `rate_limit` and retries, and their
+each time the pages of the last round have loaded. Each goes through the connection's `rate_limit`, `max_in_flight` and retries, and their
 rows load strictly in page order. The run ends at the first short (or, for `sl.page_number`, empty) page, and the
 requests already sent past it are dropped, so a table or cursor window of a few pages sends only a few requests
 more than it loads, and a long one soon has `N` open.
@@ -408,7 +408,9 @@ mock API answering each page of 100 rows in 80 ms, 1,000 pages took:
 | `read_ahead=8` | 12 s | 85 |
 | `read_ahead=20` | 6 s | 171 |
 
-Raise `read_ahead` until the API's own limits, or a `rate_limit` you set for them, become the bound.
+Raise `read_ahead` until the API's own limits, or the connection's `rate_limit` or `max_in_flight` you set for them,
+become the bound. Every table reading at once adds its own open requests, up to `concurrency` × `read_ahead`;
+`max_in_flight` caps them all together.
 
 Every job of a load shares one pool of connections, so the cursor windows of a backfill reuse the connections
 the first window opened instead of each opening its own.
@@ -550,9 +552,9 @@ def notes(ctx):
                        paginator=sl.page_number("page"), select="$.items")
 ```
 
-- Every level has its own paginator, `concurrency`, `on_status` and, through the `sl.http` it
-  comes from, its own retries and rate limit. Up to one more request than the sum of the fan-out
-  levels' `concurrency` is in flight at once (the top paginate's); a shared `rate_limit` caps them.
+- Every level has its own paginator, `concurrency` and `on_status`; retries and limits are the
+  connection's. Up to one more request than the sum of the fan-out levels' `concurrency` is in
+  flight at once (the top paginate's); the connection's `max_in_flight` and `rate_limit` cap them.
 - An incremental cursor goes on the top level, as above; lower levels only see the keys it picks.
 - Only the last level's rows load. A level that should be a table of its own is a table function
   whose job stops at that level, so the levels above it are requested again for that table.
@@ -722,15 +724,15 @@ origin, so `json_body` never leaves the connection's origin.
 
 ### OAuth2
 
-`sl.oauth2_client_credentials`, `sl.oauth2_refresh_token` and `sl.oauth2_password` fetch short-lived
-access tokens from `token_url`. See [OAuth2](/docs/guides/python/#oauth2) in the guide for an
+The `oauth2_client_credentials`, `oauth2_refresh_token` and `oauth2_password` kinds of `auth` fetch
+short-lived access tokens from `token_url`. See [OAuth2](/docs/guides/python/#oauth2) in the guide for an
 example.
 
-- The token is fetched once per run and shared by every job on the connection with the same auth.
+- The token is fetched once per run and shared by every job on the connection.
   It is renewed before it expires (by `expires_in`), and once more when the API answers 401; the
   request is then retried with the new token. When `on_status` maps 401 to `"skip"`, a 401 renews
   the token only if it is over 30 seconds old; otherwise the request is skipped.
-- `client_auth="basic"` sends `client_id` and `client_secret` to the token URL with HTTP basic
+- `client_auth = "basic"` sends `client_id` and `client_secret` to the token URL with HTTP basic
   auth, the method RFC 6749 requires servers to support. As the RFC asks, both are form-encoded
   first, so a secret with characters such as `+` or `/` reaches the server encoded. Use `"post"`
   for a server that rejects that, or that wants them as form fields.
@@ -743,7 +745,7 @@ example.
   does not save it, so each run starts from the connection's `refresh_token`. An API that revokes
   the old refresh token on rotation therefore fails on the next run; use client credentials there
   when the API offers them.
-- `sl.oauth2_password` posts `grant_type=password` with the
+- `oauth2_password` posts `grant_type=password` with the
   connection's `username` and `password`. When the response includes a `refresh_token`, later
   renewals in the run use it; when it doesn't, or the server refuses it, SQLLoad asks with the
   password again. The password is shown as `***` like the other secrets, and wrong credentials end
@@ -756,10 +758,7 @@ base_url = "https://api.example.com/v1"
 username = "loader"
 password = "${ENV:SHOP_PASSWORD}"
 client_id = "sqlload"
-```
-
-```python
-api = sl.http(shop, auth=sl.oauth2_password("https://auth.example.com/oauth/token", client_auth="post"))
+auth = { kind = "oauth2_password", token_url = "https://auth.example.com/oauth/token", client_auth = "post" }
 ```
 
 ## Logged state