Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions crates/sqlload/src/http_source/_helpers/fetch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use crate::http::models::RetryPolicy;
use crate::http_source::_helpers::urls::{link_header_next, redacted, same_origin};
use crate::http_source::constants::{
ERROR_EXCERPT_CHARS, FIRST_RETRY_DELAY, MAX_REDIRECTS, MAX_RETRY_AFTER, MAX_RETRY_DELAY,
SKIPPED_REFUSAL_RENEWAL_AGE,
SKIPPED_REFUSAL_RENEWAL_AGE, STOPPED,
};
use crate::http_source::models::{Attempt, Calls, Credential, Fetched, PageCall, StatusAction};
use crate::python_sources::errors::ReadError;
Expand All @@ -25,8 +25,15 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result<Option<Fetch
let mut refused: Option<String> = None;
let mut renewed = false;
loop {
if let Some(limiter) = &calls.limiter {
limiter.wait().await;
let waited = async {
if let Some(limiter) = &calls.limiter {
limiter.wait().await;
}
};
tokio::select! {
biased;
() = calls.stop.cancelled() => return Err(STOPPED.to_owned().into()),
() = waited => {}
}
let access = match &calls.endpoint.credential {
Credential::OAuth2(tokens) => Some(
Expand All @@ -36,6 +43,9 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result<Option<Fetch
),
_ => None,
};
if calls.stop.is_cancelled() {
return Err(STOPPED.to_owned().into());
}
match once(calls, call, access.as_deref(), !renewed).await? {
Attempt::Done(fetched) => return Ok(fetched),
Attempt::Refused => {
Expand All @@ -50,7 +60,10 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result<Option<Fetch
}
Attempt::Retry { after, .. } => {
let delay = after.unwrap_or_else(|| jittered(policy.delay(attempt)));
tokio::time::sleep(delay).await;
tokio::select! {
() = calls.stop.cancelled() => return Err(STOPPED.to_owned().into()),
() = tokio::time::sleep(delay) => {}
}
attempt += 1;
}
}
Expand Down
67 changes: 51 additions & 16 deletions crates/sqlload/src/http_source/_helpers/job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ use std::time::Duration;

use crate::http::main::install_tls_provider::install_tls_provider;
use crate::http_source::constants::{
BODY_DROPPING_REDIRECTS, JOB_THREADS, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD, SHOWN_KEYS,
BODY_DROPPING_REDIRECTS, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD, SHOWN_KEYS,
};
use crate::http_source::models::{
Auth, Calls, FanKey, FanOut, FanOutLevel, HttpJob, HttpPage, KeyFeed, Location, Method, Over,
Expand Down Expand Up @@ -572,44 +572,78 @@ fn key_problem(request: &Request, wanted: &[String], value: &Value) -> Option<St
}
}

/// Run `job` on a runtime of its own, handing each page to `deliver` in arrival order; `deliver` returning `false` cancels the requests in flight.
/// Run `job` on the shared [`jobs`] runtime, handing each page to `deliver` in arrival order; `deliver` returning `false` cancels the requests in flight.
pub(crate) fn drive_job(
job: HttpJob,
(connections, source_part): (&Map<String, Value>, Option<u64>),
mut deliver: impl FnMut(HttpPage) -> bool,
) -> Result<(), ReadError> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(JOB_THREADS)
.enable_all()
.build()
.map_err(|error| format!("cannot start the sl.http runtime: {error}"))?;
let jobs = jobs()?;
let http = match job_posts(&job) {
true => jobs.post.clone(),
false => jobs.get.clone(),
};
let (sender, mut receiver) = channel(PAGES_AHEAD);
let task = runtime.spawn(run_job(job, (connections.clone(), source_part), sender));
let task = jobs.runtime.spawn(run_job(
job,
(connections.clone(), source_part),
(http, sender),
));
while let Some(page) = receiver.blocking_recv() {
if !deliver(page) {
task.abort();
break;
}
}
drop(receiver);
match runtime.block_on(task) {
match jobs.runtime.handle().block_on(task) {
Ok(result) => result,
Err(error) if error.is_cancelled() => Ok(()),
Err(error) => Err(format!("the sl.http job stopped: {error}").into()),
}
}

/// Run `job`, sending every page to `pages`.
/// The runtime every job's requests run on, and the pooled clients they share: one for jobs that send a `POST`, one for the rest, as they follow redirects differently.
struct Jobs {
runtime: tokio::runtime::Runtime,
get: reqwest::Client,
post: reqwest::Client,
}

/// The process's [`Jobs`], made by the first job and kept: each job of a table read in cursor windows (one per window) finds the connections the jobs before it opened still in the pool, instead of a new runtime's empty one, so it opens no new connection, nor repeats a TLS handshake, to a host it already reached; a failure to make them is not kept, so the next job tries again.
fn jobs() -> Result<&'static Jobs, String> {
static JOBS: std::sync::OnceLock<Jobs> = std::sync::OnceLock::new();
if let Some(jobs) = JOBS.get() {
return Ok(jobs);
}
let threads = std::thread::available_parallelism().map_or(2, |count| count.get().max(2));
let made = Jobs {
runtime: tokio::runtime::Builder::new_multi_thread()
.worker_threads(threads)
.thread_name("sqlload-http")
.enable_all()
.build()
.map_err(|error| format!("cannot start the sl.http runtime: {error}"))?,
get: job_client(false)?,
post: job_client(true)?,
};
Ok(JOBS.get_or_init(|| made))
}

/// Whether any request of `job` sends a `POST`.
fn job_posts(job: &HttpJob) -> bool {
match job {
HttpJob::Paginate(request) => request.method == Method::Post,
HttpJob::FanOut(fan_out) => fan_out_posts(fan_out),
}
}

/// Run `job` with the pooled client `http`, sending every page to `pages`.
async fn run_job(
job: HttpJob,
(connections, source_part): (Map<String, Value>, Option<u64>),
pages: Sender<HttpPage>,
(http, pages): (reqwest::Client, Sender<HttpPage>),
) -> Result<(), ReadError> {
let posts = match &job {
HttpJob::Paginate(request) => request.method == Method::Post,
HttpJob::FanOut(fan_out) => fan_out_posts(fan_out),
};
let http = job_client(posts)?;
match job {
HttpJob::Paginate(request) => {
let calls = request_calls(&request, &connections, (&http, source_part))?;
Expand Down Expand Up @@ -697,6 +731,7 @@ fn request_calls(
on_status,
full_page_seen: std::sync::Arc::default(),
source_part,
stop: tokio_util::sync::CancellationToken::new(),
})
}

Expand Down
4 changes: 2 additions & 2 deletions crates/sqlload/src/http_source/_helpers/oauth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use crate::python_sources::errors::ReadError;
/// Every token source of this process, by connection and auth key.
static SOURCES: OnceLock<Mutex<HashMap<String, Arc<TokenSource>>>> = OnceLock::new();

/// The runtime token requests run on, and its HTTP client: one for the process, outliving any job's runtime, so a token request a cancelled page started still finishes and keeps what the server sent, such as a rotated refresh token.
/// The runtime token requests run on, and its HTTP client: one for the process, apart from the jobs' runtime, so a token request a cancelled page started still finishes and keeps what the server sent, such as a rotated refresh token.
static RENEWALS: OnceLock<Result<(tokio::runtime::Runtime, reqwest::Client), String>> =
OnceLock::new();

Expand Down Expand Up @@ -192,7 +192,7 @@ impl TokenSource {
.map_err(|error| format!("the OAuth2 token request stopped: {error}"))?
}

/// A new access token from the token URL, kept in `state` with any rotated refresh token. It runs on the process's token runtime, so a request cut short while it waits, such as a page read ahead past the last one, or the end of its job's runtime, can't drop a refresh token the server has already replaced.
/// A new access token from the token URL, kept in `state` with any rotated refresh token. It runs on the process's token runtime, so a request cut short while it waits, such as a page read ahead past the last one or an aborted job, can't drop a refresh token the server has already replaced.
async fn renewed(
&self,
http: &reqwest::Client,
Expand Down
33 changes: 29 additions & 4 deletions crates/sqlload/src/http_source/_helpers/pages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use std::collections::VecDeque;
use reqwest::Url;
use serde_json::{Map, Value};
use tokio::sync::mpsc::Sender;
use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;

use crate::http_source::_helpers::cells::pivoted_rows;
Expand All @@ -27,7 +28,7 @@ use crate::http_source::models::{
};
use crate::python_sources::errors::ReadError;

/// Send every page of `request` at `path` (its placeholders filled) to `pages`, with where its rows are; a page whose selected node is null has no rows and is not sent, and one missing it fails, naming what the response holds, unless the request reads a missing node as no rows (`select_missing`). An offset run that would end after a first page shorter than its limit asks for the next offset once, and fails when rows come back, as the API then caps its pages lower than the limit. With `read_ahead`, offset and page-number pages after a full first page are requested that many at a time (fewer under a memory budget: only as many as pages the size of the largest yet fit in a quarter of each table's share), each through the shared rate limit and retries in a task of its own (so it keeps reading while the pages' receiver holds this loop up, and is aborted when the run ends), and handled strictly in page order, so the run still ends at the first short page, dropping the requests past it. Stops quietly once `pages` closes. A next position (cursor, link or count) already requested for this request fails before it is requested again, naming the value and the page that returned it, and a counted page repeating the previous response fails, rather than loop. With `error_path`, an error a page reports there fails the table before any of that page's rows load.
/// Send every page of `request` at `path` (its placeholders filled) to `pages`, with where its rows are; a page whose selected node is null has no rows and is not sent, and one missing it fails, naming what the response holds, unless the request reads a missing node as no rows (`select_missing`). An offset run that would end after a first page shorter than its limit asks for the next offset once, and fails when rows come back, as the API then caps its pages lower than the limit. With `read_ahead`, offset and page-number pages after a full first page are requested two at a time, then four, eight and so on up to that many (fewer under a memory budget: only as many as pages the size of the largest yet fit in a quarter of each table's share), each through the shared rate limit and retries in a task of its own (so it keeps reading while the pages' receiver holds this loop up), and handled strictly in page order, so the run still ends at the first short page, dropping the requests past it. Stops quietly once `pages` closes. A next position (cursor, link or count) already requested for this request fails before it is requested again, naming the value and the page that returned it, and a counted page repeating the previous response fails, rather than loop. With `error_path`, an error a page reports there fails the table before any of that page's rows load.
pub(crate) async fn paginate(
calls: &Calls,
(request, context): (&Request, &RowContext),
Expand All @@ -49,8 +50,11 @@ pub(crate) async fn paginate(
let mut names: Option<CellNames> = None;
let ahead = read_ahead(&request.paginator);
let mut in_flight: VecDeque<AbortOnDropHandle<AheadPage>> = VecDeque::new();
let stop = CancellationToken::new();
let _stopping = stop.clone().drop_guard();
let mut queued: u64 = 1;
let mut largest: u64 = 0;
let (mut ramp, mut ramped): (u64, u64) = (1, 1);
for step in 1_u64.. {
let shown = redacted(
&format!("{} {}", call.method.as_str(), call.url),
Expand All @@ -63,10 +67,17 @@ pub(crate) async fn paginate(
(Some(part), largest) if largest > 0 => (part / largest).max(1),
_ => ahead,
};
while queued < step + ahead.min(fitting) - 1 {
let grows = step > ramped;
if grows {
ramp = ramp.saturating_mul(2).min(ahead);
}
while queued < step + ramp.min(fitting) - 1 {
queued += 1;
let ahead_call = counted_call(request, &first, queued)?;
let ahead_calls = calls.clone();
let ahead_calls = Calls {
stop: stop.clone(),
..calls.clone()
};
in_flight.push_back(AbortOnDropHandle::new(tokio::spawn(async move {
let fetched = fetch(&ahead_calls, &ahead_call).await;
AheadPage {
Expand All @@ -75,6 +86,9 @@ pub(crate) async fn paginate(
}
})));
}
if grows {
ramped = queued;
}
let next =
match in_flight.pop_front() {
Some(handle) => Some(handle.await.map_err(|error| {
Expand All @@ -94,6 +108,7 @@ pub(crate) async fn paginate(
}
};
let Some(fetched) = fetched else {
release(in_flight);
return Ok(());
};
largest = largest.max(fetched.bytes.len() as u64);
Expand Down Expand Up @@ -171,12 +186,22 @@ pub(crate) async fn paginate(
}
match next {
Some((next, _)) => call = next,
None => return Ok(()),
None => {
release(in_flight);
return Ok(());
}
}
}
Ok(())
}

/// Let the pages still read ahead when paging ends finish their requests, so their connections go back to the pool, instead of aborting them with the paging, as a failure or a closed `pages` does; the paging's stop token keeps them from waiting for a turn or retrying.
fn release(in_flight: VecDeque<AbortOnDropHandle<AheadPage>>) {
for handle in in_flight {
drop(handle.detach());
}
}

/// What identifies a request among those a paginator sent: its URL and body.
fn seen_key(call: &PageCall) -> (Url, Option<String>) {
(call.url.clone(), call.body.as_ref().map(Value::to_string))
Expand Down
6 changes: 3 additions & 3 deletions crates/sqlload/src/http_source/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ pub const REPORTED_ERROR_CHARS: usize = 200;
/// Pages fetched ahead of the parser before a job waits.
pub const PAGES_AHEAD: usize = 4;

/// Threads of the runtime one job's requests run on.
pub const JOB_THREADS: usize = 2;

/// The `Link` header parameter naming relations.
pub const RELATION_PARAMETER: &str = "rel";

Expand Down Expand Up @@ -65,3 +62,6 @@ pub const SHOWN_KEYS: usize = 5;

/// The most pages `read_ahead` keeps requested at once.
pub const MAX_READ_AHEAD: u64 = 100;

/// Why a page requested ahead ended without an answer: the paging stopped before it needed the page.
pub const STOPPED: &str = "the paging stopped before it needed this page";
4 changes: 3 additions & 1 deletion crates/sqlload/src/http_source/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ pub enum Paginator {
limit: u64,
#[serde(default)]
location: Location,
/// Pages requested at once after a full first page, their rows still loaded in page order; `None` reads one at a time.
/// The most pages requested at once after a full first page, ramped up to it (2, 4, 8 …), their rows still loaded in page order; `None` reads one at a time.
#[serde(default)]
read_ahead: Option<u64>,
},
Expand Down Expand Up @@ -450,6 +450,8 @@ pub struct Calls {
pub full_page_seen: std::sync::Arc<std::sync::atomic::AtomicBool>,
/// The most the pages `read_ahead` requests may hold at once: a quarter of the table's share of the run's memory budget; `None` without a budget.
pub source_part: Option<u64>,
/// Cancelled once the pages read ahead are no longer wanted: after paging ends normally a request already sent still finishes, so its connection goes back to the pool, but no wait for a turn, retry or other attempt starts.
pub stop: tokio_util::sync::CancellationToken,
}

/// What one try at a request came to.
Expand Down
Loading
Loading