Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 75 additions & 42 deletions crates/sqlload/src/http_source/_helpers/endpoint.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! A connection's settings resolved into what requests need, and the rate limit its jobs share.
//! A connection's settings resolved into what requests need, and the limits every request to it shares.

use crate::load_events::main::emit::emit;
use crate::load_events::models::EventKind;
Expand All @@ -12,52 +12,70 @@ use serde_json::{Map, Value};
use crate::http_source::_helpers::oauth::shared_token_source;
use crate::http_source::_helpers::urls::encoded;
use crate::http_source::constants::SECRET_HEADER_WORDS;
use crate::http_source::models::{Auth, ClientSpec, Credential, Endpoint, Grant, Limiter};
use crate::http_source::models::{
Auth, ClientSpec, ConnectionLimits, Credential, Endpoint, Grant, HttpSettings, Limiter,
};

/// The settings of `client`'s connection that requests share, and the connection's own settings.
pub(crate) fn connection_settings<'a>(
client: &ClientSpec,
connections: &'a Map<String, Value>,
) -> Result<(HttpSettings, &'a Map<String, Value>), String> {
let name = &client.connection;
let settings = connections
.get(name)
.and_then(Value::as_object)
.ok_or_else(|| {
format!("sl.http uses connection {name}, which is not in sqlload.toml or is a target's connection")
})?;
let shared = serde_json::from_value::<HttpSettings>(Value::Object(settings.clone()))
.map_err(|error| format!("[connections.{name}]: {error}"))?;
Ok((shared, settings))
}

/// The endpoint of `client`'s connection: its `base_url`, headers and the credential its `auth` names.
pub(crate) fn endpoint(
client: &ClientSpec,
connections: &Map<String, Value>,
(auth, settings): (Option<&Auth>, &Map<String, Value>),
) -> Result<Endpoint, String> {
let name = &client.connection;
let settings = connections.get(name).ok_or_else(|| {
format!("sl.http uses connection {name}, which is not in sqlload.toml or is a target's connection")
})?;
let text = |key: &str| settings.get(key).and_then(Value::as_str).map(str::to_owned);
let base =
text("base_url").ok_or_else(|| format!("connection {name} needs base_url for sl.http"))?;
let base_url = reqwest::Url::parse(&base)
.map_err(|error| format!("connection {name}: base_url {base} is not a URL: {error}"))?;
let needed = |key: &str| {
text(key).filter(|value| !value.is_empty()).ok_or_else(|| {
format!("connection {name} needs {key} for the sl.http auth this job uses")
})
text(key)
.filter(|value| !value.is_empty())
.ok_or_else(|| format!("connection {name} needs {key} for its auth"))
};
let present = |key: &str| text(key).filter(|value| !value.is_empty());
let credential = match &client.auth {
Auth::None => Credential::None,
Auth::Bearer => Credential::Header {
let credential = match auth {
None => Credential::None,
Some(Auth::Bearer {}) => Credential::Header {
name: "Authorization".to_owned(),
value: format!("Bearer {}", needed("token")?),
},
Auth::ApiKey {
Some(Auth::ApiKey {
header: Some(header),
query: None,
} => Credential::Header {
}) => Credential::Header {
name: header.clone(),
value: needed("token")?,
},
Auth::ApiKey {
Some(Auth::ApiKey {
header: None,
query: Some(query),
} => Credential::Query {
}) => Credential::Query {
name: query.clone(),
value: needed("token")?,
},
Auth::ApiKey { .. } => {
return Err("sl.api_key takes exactly one of header= or query=".to_owned());
Some(Auth::ApiKey { .. }) => {
return Err(format!(
"[connections.{name}] auth api_key takes exactly one of header or query"
));
}
Auth::Basic => {
Some(Auth::Basic {}) => {
let pair = format!("{}:{}", needed("username")?, needed("password")?);
Credential::Header {
name: "Authorization".to_owned(),
Expand All @@ -67,39 +85,39 @@ pub(crate) fn endpoint(
),
}
}
Auth::Oauth2ClientCredentials(spec) => {
Some(Auth::Oauth2ClientCredentials(spec)) => {
let grant = Grant::ClientCredentials {
client_id: needed("client_id")?,
client_secret: needed("client_secret")?,
};
Credential::OAuth2(shared_token_source(
name,
&format!("{:?}", client.auth),
&format!("{auth:?}"),
spec,
grant,
)?)
}
Auth::Oauth2RefreshToken(spec) => {
Some(Auth::Oauth2RefreshToken(spec)) => {
let grant = Grant::RefreshToken {
refresh_token: needed("refresh_token")?,
client: (present("client_id"), present("client_secret")),
};
Credential::OAuth2(shared_token_source(
name,
&format!("{:?}", client.auth),
&format!("{auth:?}"),
spec,
grant,
)?)
}
Auth::Oauth2Password(spec) => {
Some(Auth::Oauth2Password(spec)) => {
let grant = Grant::Password {
username: needed("username")?,
password: needed("password")?,
client: (present("client_id"), present("client_secret")),
};
Credential::OAuth2(shared_token_source(
name,
&format!("{:?}", client.auth),
&format!("{auth:?}"),
spec,
grant,
)?)
Expand Down Expand Up @@ -179,27 +197,42 @@ fn text_headers(headers: &Map<String, Value>) -> BTreeMap<String, String> {
.collect()
}

/// The limiter every job of `connection` with the limit `rate` (`10/s`, `600/m`, `1000/h`) shares in this process.
pub(crate) fn shared_limiter(connection: &str, rate: &str) -> Result<Arc<Limiter>, String> {
static LIMITERS: OnceLock<Mutex<HashMap<String, Arc<Limiter>>>> = OnceLock::new();
let interval = interval(rate)?;
let limiters = LIMITERS.get_or_init(Mutex::default);
let mut limiters = limiters
/// The limiter and in-flight cap every request to `connection` shares in this process, made for its `limits` by the first job to call it. A later project in the same process (as tests run) that gives the connection other limits replaces them.
pub(crate) fn shared_limits(
connection: &str,
limits: &ConnectionLimits,
) -> Result<SharedLimits, String> {
static SHARED: OnceLock<Mutex<HashMap<String, (ConnectionLimits, SharedLimits)>>> =
OnceLock::new();
let mut shared = SHARED
.get_or_init(Mutex::default)
.lock()
.map_err(|_| "the rate limiters' lock is poisoned".to_owned())?;
let limiter = limiters
.entry(format!("{connection}\u{0}{rate}"))
.or_insert_with(|| {
Arc::new(Limiter {
connection: connection.to_owned(),
rate: rate.to_owned(),
interval,
next: Mutex::new(tokio::time::Instant::now()),
})
});
Ok(limiter.clone())
if let Some((made_for, made)) = shared.get(connection)
&& made_for == limits
{
return Ok(made.clone());
}
let limiter = match &limits.rate_limit {
Some(rate) => Some(Arc::new(Limiter {
connection: connection.to_owned(),
rate: rate.clone(),
interval: interval(rate)?,
next: Mutex::new(tokio::time::Instant::now()),
})),
None => None,
};
let in_flight = limits
.max_in_flight
.map(|most| Arc::new(tokio::sync::Semaphore::new(most)));
let made = (limiter, in_flight);
shared.insert(connection.to_owned(), (limits.clone(), made.clone()));
Ok(made)
}

/// A connection's shared limiter and in-flight cap, each `None` when it sets no such limit.
pub(crate) type SharedLimits = (Option<Arc<Limiter>>, Option<Arc<tokio::sync::Semaphore>>);

/// The time between requests `rate` allows.
pub(crate) fn interval(rate: &str) -> Result<Duration, String> {
let invalid =
Expand Down
28 changes: 23 additions & 5 deletions crates/sqlload/src/http_source/_helpers/fetch.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! One request with the shared rate limit, retries with backoff, `Retry-After`, and `on_status`.
//! One request under the connection's shared limits, retries with backoff, `Retry-After`, and `on_status`.

use std::time::{Duration, SystemTime, UNIX_EPOCH};

Expand All @@ -14,7 +14,7 @@ use crate::http_source::constants::{
use crate::http_source::models::{Attempt, Calls, Credential, Fetched, PageCall, StatusAction};
use crate::python_sources::errors::ReadError;

/// The response to `call`, its JSON body resent on every retry, or `None` when `on_status` skips its status; transport errors, 429 and 5xx are retried up to `calls.retries` times, and are still worth retrying later once those run out.
/// The response to `call`, its JSON body resent on every retry, or `None` when `on_status` skips its status; transport errors, 429 and 5xx are retried up to `calls.retries` times, and are still worth retrying later once those run out. Every attempt, retries included, holds one of the connection's `max_in_flight` permits while its request is open and takes a turn under its `rate_limit`. The permit comes first, then the OAuth2 access token, then the turn, so a request that got its turn starts at once: neither a queue for a permit nor a token fetch lets turns pass unused and then go in a burst. The token is taken again after the turn, which returns the one just fetched unless the wait outlasted it. The permit is released once the response is read, not held through a retry's backoff.
pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result<Option<Fetched>, ReadError> {
let policy = RetryPolicy {
first_delay: FIRST_RETRY_DELAY,
Expand All @@ -25,15 +25,31 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result<Option<Fetch
let mut refused: Option<String> = None;
let mut renewed = false;
loop {
let waited = async {
let acquired = async {
match &calls.in_flight {
Some(in_flight) => Some(in_flight.clone().acquire_owned().await),
None => None,
}
};
let permit = tokio::select! {
biased;
() = calls.stop.cancelled() => return Err(STOPPED.to_owned().into()),
permit = acquired => permit,
};
if let Credential::OAuth2(tokens) = &calls.endpoint.credential {
tokens
.access_token(calls.retries, refused.as_deref())
.await?;
}
let turn = async {
if let Some(limiter) = &calls.limiter {
limiter.wait().await;
}
};
tokio::select! {
biased;
() = calls.stop.cancelled() => return Err(STOPPED.to_owned().into()),
() = waited => {}
() = turn => {}
}
let access = match &calls.endpoint.credential {
Credential::OAuth2(tokens) => Some(
Expand All @@ -46,7 +62,9 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result<Option<Fetch
if calls.stop.is_cancelled() {
return Err(STOPPED.to_owned().into());
}
match once(calls, call, access.as_deref(), !renewed).await? {
let attempted = once(calls, call, access.as_deref(), !renewed).await?;
drop(permit);
match attempted {
Attempt::Done(fetched) => return Ok(fetched),
Attempt::Refused => {
refused = access;
Expand Down
33 changes: 9 additions & 24 deletions crates/sqlload/src/http_source/_helpers/job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ use tokio::sync::mpsc::{Receiver, Sender, channel};
use tokio::task::JoinSet;

use crate::http_source::_helpers::cells::cells_problem;
use crate::http_source::_helpers::endpoint::{endpoint, interval, shared_limiter};
use crate::http_source::_helpers::oauth::token_url;
use crate::http_source::_helpers::endpoint::{connection_settings, endpoint, shared_limits};
use crate::http_source::_helpers::pages::{paginate, set_at};
use crate::http_source::_helpers::reshaped::reshapes;
use crate::http_source::_helpers::selector::{enclosing_levels, selected_rows, selector};
Expand All @@ -28,10 +27,11 @@ use std::time::Duration;

use crate::http::main::install_tls_provider::install_tls_provider;
use crate::http_source::constants::{
BODY_DROPPING_REDIRECTS, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD, SHOWN_KEYS,
BODY_DROPPING_REDIRECTS, DEFAULT_RETRIES, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD,
SHOWN_KEYS,
};
use crate::http_source::models::{
Auth, Calls, FanKey, FanOut, FanOutLevel, HttpJob, HttpPage, KeyFeed, Location, Method, Over,
Calls, FanKey, FanOut, FanOutLevel, HttpJob, HttpPage, KeyFeed, Location, Method, Over,
Paginator, Request, RowContext, Selector, Sink, StatusAction,
};

Expand Down Expand Up @@ -188,26 +188,13 @@ fn request_problems(request: &Request) -> Vec<String> {
problems.extend(body_problems(request));
problems.extend(request.cells.as_ref().and_then(cells_problem));
problems.extend(request.error_path.as_deref().and_then(error_path_problem));
problems.extend(match &request.client.auth {
Auth::Oauth2ClientCredentials(spec)
| Auth::Oauth2RefreshToken(spec)
| Auth::Oauth2Password(spec) => token_url(spec).err(),
Auth::None | Auth::Bearer | Auth::ApiKey { .. } | Auth::Basic => None,
});
problems.extend(
request
.client
.headers
.iter()
.filter_map(|(name, value)| header_problem(name, value)),
);
problems.extend(
request
.client
.rate_limit
.as_deref()
.and_then(|rate| interval(rate).err()),
);
problems.extend(
request
.on_status
Expand Down Expand Up @@ -708,11 +695,8 @@ fn request_calls(
(http, source_part): (&reqwest::Client, Option<u64>),
) -> Result<Calls, String> {
let client = &request.client;
let limiter = client
.rate_limit
.as_deref()
.map(|rate| shared_limiter(&client.connection, rate))
.transpose()?;
let (shared, settings) = connection_settings(client, connections)?;
let (limiter, in_flight) = shared_limits(&client.connection, &shared.limits)?;
let on_status = request
.on_status
.iter()
Expand All @@ -725,9 +709,10 @@ fn request_calls(
.collect::<Result<BTreeMap<u16, StatusAction>, String>>()?;
Ok(Calls {
http: http.clone(),
endpoint: endpoint(client, connections)?,
endpoint: endpoint(client, (shared.auth.as_ref(), settings))?,
limiter,
retries: client.retries,
in_flight,
retries: shared.retries.unwrap_or(DEFAULT_RETRIES),
on_status,
full_page_seen: std::sync::Arc::default(),
source_part,
Expand Down
9 changes: 9 additions & 0 deletions crates/sqlload/src/http_source/constants.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
use std::time::Duration;

/// Retries of a failed request when the connection sets no `retries`.
pub const DEFAULT_RETRIES: u32 = 5;

/// The most retries a connection may set.
pub const MAX_RETRIES: u32 = 100;

/// The largest `max_in_flight` a connection may set.
pub const MAX_IN_FLIGHT: usize = 10_000;

/// The delay before the first retry; each later one doubles it.
pub const FIRST_RETRY_DELAY: Duration = Duration::from_millis(500);

Expand Down
Loading
Loading