From 1e1ca7bf4d99e95792429fc20629b80638215e9a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:05:05 +0000 Subject: [PATCH 1/4] perf(http): reuse connections across cursor windows and ramp up read_ahead Every sl.http job now runs on one process-wide runtime with one pooled client (one for jobs that POST), so the cursor windows of a backfill reuse the connections the first window opened instead of each window starting a runtime and client of its own and handshaking again. read_ahead now ramps up: after a full first page, two pages are requested at a time, then four, eight and so on up to read_ahead, so a window of a few pages sends only a few requests past its end. Pages read ahead that are no longer needed are no longer aborted mid-request, which closed their pooled connection; they stop before any further wait or retry instead. --- .../sqlload/src/http_source/_helpers/fetch.rs | 18 ++- .../sqlload/src/http_source/_helpers/job.rs | 64 ++++++-- .../sqlload/src/http_source/_helpers/pages.rs | 26 ++- crates/sqlload/src/http_source/constants.rs | 6 +- crates/sqlload/src/http_source/models.rs | 4 +- .../sqlload/tests/test_http_connections.py | 151 ++++++++++++++++++ packages/sqlload/tests/test_http_jobs.py | 2 +- .../src/content/docs/docs/sources/python.mdx | 11 +- 8 files changed, 248 insertions(+), 34 deletions(-) create mode 100644 packages/sqlload/tests/test_http_connections.py diff --git a/crates/sqlload/src/http_source/_helpers/fetch.rs b/crates/sqlload/src/http_source/_helpers/fetch.rs index 9ea790a20..8d49771fd 100644 --- a/crates/sqlload/src/http_source/_helpers/fetch.rs +++ b/crates/sqlload/src/http_source/_helpers/fetch.rs @@ -9,7 +9,7 @@ use crate::http::models::RetryPolicy; use crate::http_source::_helpers::urls::{link_header_next, redacted, same_origin}; use crate::http_source::constants::{ ERROR_EXCERPT_CHARS, FIRST_RETRY_DELAY, MAX_REDIRECTS, MAX_RETRY_AFTER, MAX_RETRY_DELAY, - SKIPPED_REFUSAL_RENEWAL_AGE, + SKIPPED_REFUSAL_RENEWAL_AGE, STOPPED, }; use crate::http_source::models::{Attempt, Calls, Credential, Fetched, PageCall, StatusAction}; use crate::python_sources::errors::ReadError; @@ -25,8 +25,15 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result = None; let mut renewed = false; loop { - if let Some(limiter) = &calls.limiter { - limiter.wait().await; + let waited = async { + if let Some(limiter) = &calls.limiter { + limiter.wait().await; + } + }; + tokio::select! { + biased; + () = calls.stop.cancelled() => return Err(STOPPED.to_owned().into()), + () = waited => {} } let access = match &calls.endpoint.credential { Credential::OAuth2(tokens) => Some( @@ -50,7 +57,10 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result { let delay = after.unwrap_or_else(|| jittered(policy.delay(attempt))); - tokio::time::sleep(delay).await; + tokio::select! { + () = calls.stop.cancelled() => return Err(STOPPED.to_owned().into()), + () = tokio::time::sleep(delay) => {} + } attempt += 1; } } diff --git a/crates/sqlload/src/http_source/_helpers/job.rs b/crates/sqlload/src/http_source/_helpers/job.rs index 4e3b90c58..cbafd9f81 100644 --- a/crates/sqlload/src/http_source/_helpers/job.rs +++ b/crates/sqlload/src/http_source/_helpers/job.rs @@ -28,7 +28,7 @@ use std::time::Duration; use crate::http::main::install_tls_provider::install_tls_provider; use crate::http_source::constants::{ - BODY_DROPPING_REDIRECTS, JOB_THREADS, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD, SHOWN_KEYS, + BODY_DROPPING_REDIRECTS, MAX_READ_AHEAD, MAX_REDIRECTS, PAGES_AHEAD, SHOWN_KEYS, }; use crate::http_source::models::{ Auth, Calls, FanKey, FanOut, FanOutLevel, HttpJob, HttpPage, KeyFeed, Location, Method, Over, @@ -572,19 +572,21 @@ fn key_problem(request: &Request, wanted: &[String], value: &Value) -> Option, Option), mut deliver: impl FnMut(HttpPage) -> bool, ) -> Result<(), ReadError> { - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(JOB_THREADS) - .enable_all() - .build() - .map_err(|error| format!("cannot start the sl.http runtime: {error}"))?; + let jobs = jobs()?; + let http = match job_posts(&job) { + true => jobs.post.clone(), + false => jobs.get.clone(), + }; let (sender, mut receiver) = channel(PAGES_AHEAD); - let task = runtime.spawn(run_job(job, (connections.clone(), source_part), sender)); + let task = jobs + .runtime + .spawn(run_job(job, (connections.clone(), source_part), (http, sender))); while let Some(page) = receiver.blocking_recv() { if !deliver(page) { task.abort(); @@ -592,24 +594,53 @@ pub(crate) fn drive_job( } } drop(receiver); - match runtime.block_on(task) { + match jobs.runtime.handle().block_on(task) { Ok(result) => result, Err(error) if error.is_cancelled() => Ok(()), Err(error) => Err(format!("the sl.http job stopped: {error}").into()), } } -/// Run `job`, sending every page to `pages`. +/// The runtime every job's requests run on, and the pooled clients they share: one for jobs that send a `POST`, one for the rest, as they follow redirects differently. +struct Jobs { + runtime: tokio::runtime::Runtime, + get: reqwest::Client, + post: reqwest::Client, +} + +/// The process's [`Jobs`], made by the first job and kept: each job of a table read in cursor windows (one per window) finds the connections the jobs before it opened still in the pool, instead of a new runtime's empty one, so it opens no new connection, nor repeats a TLS handshake, to a host it already reached. +fn jobs() -> Result<&'static Jobs, String> { + static JOBS: std::sync::OnceLock> = std::sync::OnceLock::new(); + JOBS.get_or_init(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .thread_name("sqlload-http") + .enable_all() + .build() + .map_err(|error| format!("cannot start the sl.http runtime: {error}"))?; + Ok(Jobs { + runtime, + get: job_client(false)?, + post: job_client(true)?, + }) + }) + .as_ref() + .map_err(Clone::clone) +} + +/// Whether any request of `job` sends a `POST`. +fn job_posts(job: &HttpJob) -> bool { + match job { + HttpJob::Paginate(request) => request.method == Method::Post, + HttpJob::FanOut(fan_out) => fan_out_posts(fan_out), + } +} + +/// Run `job` with the pooled client `http`, sending every page to `pages`. async fn run_job( job: HttpJob, (connections, source_part): (Map, Option), - pages: Sender, + (http, pages): (reqwest::Client, Sender), ) -> Result<(), ReadError> { - let posts = match &job { - HttpJob::Paginate(request) => request.method == Method::Post, - HttpJob::FanOut(fan_out) => fan_out_posts(fan_out), - }; - let http = job_client(posts)?; match job { HttpJob::Paginate(request) => { let calls = request_calls(&request, &connections, (&http, source_part))?; @@ -697,6 +728,7 @@ fn request_calls( on_status, full_page_seen: std::sync::Arc::default(), source_part, + stop: tokio_util::sync::CancellationToken::new(), }) } diff --git a/crates/sqlload/src/http_source/_helpers/pages.rs b/crates/sqlload/src/http_source/_helpers/pages.rs index aa5281395..93dd533a1 100644 --- a/crates/sqlload/src/http_source/_helpers/pages.rs +++ b/crates/sqlload/src/http_source/_helpers/pages.rs @@ -10,7 +10,8 @@ use std::collections::VecDeque; use reqwest::Url; use serde_json::{Map, Value}; use tokio::sync::mpsc::Sender; -use tokio_util::task::AbortOnDropHandle; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; use crate::http_source::_helpers::cells::pivoted_rows; use crate::http_source::_helpers::fetch::fetch; @@ -48,9 +49,12 @@ pub(crate) async fn paginate( let mut previous: Option = None; let mut names: Option = None; let ahead = read_ahead(&request.paginator); - let mut in_flight: VecDeque> = VecDeque::new(); + let mut in_flight: VecDeque> = VecDeque::new(); + let stop = CancellationToken::new(); + let _stopping = stop.clone().drop_guard(); let mut queued: u64 = 1; let mut largest: u64 = 0; + let (mut ramp, mut ramped): (u64, u64) = (1, 1); for step in 1_u64.. { let shown = redacted( &format!("{} {}", call.method.as_str(), call.url), @@ -63,17 +67,27 @@ pub(crate) async fn paginate( (Some(part), largest) if largest > 0 => (part / largest).max(1), _ => ahead, }; - while queued < step + ahead.min(fitting) - 1 { + let grows = step > ramped; + if grows { + ramp = ramp.saturating_mul(2).min(ahead); + } + while queued < step + ramp.min(fitting) - 1 { queued += 1; let ahead_call = counted_call(request, &first, queued)?; - let ahead_calls = calls.clone(); - in_flight.push_back(AbortOnDropHandle::new(tokio::spawn(async move { + let ahead_calls = Calls { + stop: stop.clone(), + ..calls.clone() + }; + in_flight.push_back(tokio::spawn(async move { let fetched = fetch(&ahead_calls, &ahead_call).await; AheadPage { call: ahead_call, fetched, } - }))); + })); + } + if grows { + ramped = queued; } let next = match in_flight.pop_front() { diff --git a/crates/sqlload/src/http_source/constants.rs b/crates/sqlload/src/http_source/constants.rs index 7a23a75d9..ad58c7b0d 100644 --- a/crates/sqlload/src/http_source/constants.rs +++ b/crates/sqlload/src/http_source/constants.rs @@ -27,9 +27,6 @@ pub const REPORTED_ERROR_CHARS: usize = 200; /// Pages fetched ahead of the parser before a job waits. pub const PAGES_AHEAD: usize = 4; -/// Threads of the runtime one job's requests run on. -pub const JOB_THREADS: usize = 2; - /// The `Link` header parameter naming relations. pub const RELATION_PARAMETER: &str = "rel"; @@ -65,3 +62,6 @@ pub const SHOWN_KEYS: usize = 5; /// The most pages `read_ahead` keeps requested at once. pub const MAX_READ_AHEAD: u64 = 100; + +/// Why a page requested ahead ended without an answer: the paging stopped before it needed the page. +pub const STOPPED: &str = "the paging stopped before it needed this page"; diff --git a/crates/sqlload/src/http_source/models.rs b/crates/sqlload/src/http_source/models.rs index 8712801db..551b418e6 100644 --- a/crates/sqlload/src/http_source/models.rs +++ b/crates/sqlload/src/http_source/models.rs @@ -344,7 +344,7 @@ pub enum Paginator { limit: u64, #[serde(default)] location: Location, - /// Pages requested at once after a full first page, their rows still loaded in page order; `None` reads one at a time. + /// The most pages requested at once after a full first page, ramped up to it (2, 4, 8 …), their rows still loaded in page order; `None` reads one at a time. #[serde(default)] read_ahead: Option, }, @@ -450,6 +450,8 @@ pub struct Calls { pub full_page_seen: std::sync::Arc, /// The most the pages `read_ahead` requests may hold at once: a quarter of the table's share of the run's memory budget; `None` without a budget. pub source_part: Option, + /// Cancelled once the pages read ahead are no longer wanted: a request already sent still finishes, so its connection goes back to the pool, but no wait for a turn, retry or other attempt starts. + pub stop: tokio_util::sync::CancellationToken, } /// What one try at a request came to. diff --git a/packages/sqlload/tests/test_http_connections.py b/packages/sqlload/tests/test_http_connections.py new file mode 100644 index 000000000..e01766cdb --- /dev/null +++ b/packages/sqlload/tests/test_http_connections.py @@ -0,0 +1,151 @@ +"""How sl.http jobs use connections: cursor windows share one pool of HTTPS connections, and read_ahead ramps up instead of +sending every request at once. The stub API serves over TLS with a certificate from a CA made for the test, and counts the +TLS handshakes and requests it sees.""" + +import json +import os +import ssl +import subprocess +import sys +import textwrap +import threading +from collections.abc import Iterator +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +import duckdb +import pytest + +DAYS = 12 +ROWS_PER_DAY = 5 +#: Midnight UTC twelve days ago: the first window's start, so the windows of one day each end at today's. +START = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=DAYS) +ROWS = [{"id": n + 1, "updated_at": (START + timedelta(days=1) / ROWS_PER_DAY * n).isoformat()} for n in range(DAYS * ROWS_PER_DAY)] + + +class Api(BaseHTTPRequestHandler): + """`/items?updated_from&updated_to&offset&limit`: the rows in that range, `limit` from `offset`; without a range, every row.""" + + protocol_version = "HTTP/1.1" + + def log_message(self, *args: Any) -> None: + pass + + def do_GET(self) -> None: + query = {name: values[0] for name, values in parse_qs(urlparse(self.path).query).items()} + low = datetime.fromisoformat(query["updated_from"]) if "updated_from" in query else START + high = datetime.fromisoformat(query["updated_to"]) if "updated_to" in query else START + timedelta(days=DAYS) + ranged = [row for row in ROWS if low.astimezone(timezone.utc) <= datetime.fromisoformat(row["updated_at"]) < high.astimezone(timezone.utc)] + offset, limit = int(query.get("offset", 0)), int(query["limit"]) + with self.server.lock: # type: ignore[attr-defined] + self.server.requests += 1 # type: ignore[attr-defined] + data = json.dumps({"items": ranged[offset : offset + limit]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +class TlsServer(ThreadingHTTPServer): + """Serves `Api` over TLS, each connection's handshake in its own thread, counting the handshakes that succeed.""" + + daemon_threads = True + + def finish_request(self, request: Any, client_address: Any) -> None: + try: + tls = self.context.wrap_socket(request, server_side=True) # type: ignore[attr-defined] + except (ssl.SSLError, OSError): + return + with self.lock: # type: ignore[attr-defined] + self.handshakes += 1 # type: ignore[attr-defined] + with tls: + super().finish_request(tls, client_address) + + +@pytest.fixture(scope="module") +def certificates(tmp_path_factory: pytest.TempPathFactory) -> Path: + """A CA, and a certificate it signed for `localhost` with its key.""" + folder = tmp_path_factory.mktemp("tls") + + def openssl(*args: str) -> None: + subprocess.run(["openssl", *args], cwd=folder, check=True, capture_output=True) + + openssl("req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", "ca.key", "-out", "ca.pem", "-days", "2", "-subj", "/CN=test CA", + "-addext", "basicConstraints=critical,CA:TRUE", "-addext", "keyUsage=critical,keyCertSign") + openssl("req", "-newkey", "rsa:2048", "-nodes", "-keyout", "server.key", "-out", "server.csr", "-subj", "/CN=localhost") + (folder / "ext.cnf").write_text("subjectAltName=DNS:localhost\nbasicConstraints=CA:FALSE\nextendedKeyUsage=serverAuth\n") + openssl("x509", "-req", "-in", "server.csr", "-CA", "ca.pem", "-CAkey", "ca.key", "-CAcreateserial", "-out", "server.pem", "-days", "2", "-extfile", "ext.cnf") + return folder + + +@pytest.fixture +def api(certificates: Path) -> Iterator[TlsServer]: + server = TlsServer(("127.0.0.1", 0), Api) + server.context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) # type: ignore[attr-defined] + server.context.load_cert_chain(certificates / "server.pem", certificates / "server.key") # type: ignore[attr-defined] + server.lock, server.handshakes, server.requests = threading.Lock(), 0, 0 # type: ignore[attr-defined] + threading.Thread(target=server.serve_forever, daemon=True).start() + yield server + server.shutdown() + + +def load(tmp_path: Path, api: TlsServer, certificates: Path, source: str, job: str) -> subprocess.CompletedProcess[str]: + """Load table `items` of source `source` (its `sl.source` arguments), returning `job`, from the stub into DuckDB, trusting the test's CA.""" + port = api.server_address[1] + (tmp_path / "sources").mkdir() + (tmp_path / "home").mkdir() + (tmp_path / "sqlload.toml").write_text( + textwrap.dedent(f"""\ + python_paths = ["sources"] + + [connections.wh] + kind = "duckdb" + database = "wh.duckdb" + + [connections.api] + kind = "http" + base_url = "https://localhost:{port}" + + [targets.dev] + connection = "wh" + loader_schema = "raw" + """) + ) + (tmp_path / "sources" / "api.py").write_text( + f"import sqlload as sl\n\napi = sl.source('api', connection='api', {source})\nhttp = sl.http(api)\n\n" + f"@api.table(description='Items.')\ndef items(ctx):\n return {job}\n" + ) + env = {**os.environ, "HOME": str(tmp_path / "home"), "SSL_CERT_FILE": str(certificates / "ca.pem")} + return subprocess.run([sys.executable, "-c", "from sqlload._cli import main; main()", "load"], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=120) + + +def loaded(tmp_path: Path) -> list[tuple[Any, ...]]: + with duckdb.connect(str(tmp_path / "wh.duckdb"), read_only=True) as connection: + return connection.execute("SELECT count(*), count(DISTINCT id) FROM raw.api__items").fetchall() + + +def test_given_one_day_cursor_windows_over_https_when_loading_then_the_windows_share_the_connections_the_first_opened( + tmp_path: Path, api: TlsServer, certificates: Path +) -> None: + # Thirteen windows, each three pages of two rows, two of them read ahead together: one pool of connections holds them all. + source = f"write_strategy='append', cursor_column='updated_at', cursor_start='{START:%Y-%m-%d %H:%M:%S}', batch_size='1d'" + job = "http.paginate('/items', params={'updated_from': ctx.cursor_value, 'updated_to': ctx.cursor_end}, paginator=sl.offset(limit=2, read_ahead=4), select='$.items')" + done = load(tmp_path, api, certificates, source, job) + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(tmp_path) == [(len(ROWS), len(ROWS))] + assert api.handshakes <= 4, api.handshakes # type: ignore[attr-defined] + + +def test_given_read_ahead_of_twenty_when_a_table_has_few_pages_then_requests_ramp_up_and_stay_close_to_the_pages_read( + tmp_path: Path, api: TlsServer, certificates: Path +) -> None: + # Twelve rows over six full pages of two, then an empty page ends the paging: seven pages needed, where twenty at once would send 26. + job = "http.paginate('/items', paginator=sl.offset(limit=2, read_ahead=20), select='$.items')" + done = load(tmp_path, api, certificates, "write_strategy='replace'", job.replace("'/items'", "'/items', params={'updated_to': '" + (START + timedelta(days=2, hours=9)).isoformat() + "'}")) + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(tmp_path) == [(12, 12)] + assert api.requests <= 7 + 4, api.requests # type: ignore[attr-defined] diff --git a/packages/sqlload/tests/test_http_jobs.py b/packages/sqlload/tests/test_http_jobs.py index 4ddcead56..322e8545b 100644 --- a/packages/sqlload/tests/test_http_jobs.py +++ b/packages/sqlload/tests/test_http_jobs.py @@ -928,7 +928,7 @@ def test_given_read_ahead_when_later_pages_answer_first_and_one_is_retried_then_ assert done.returncode == 0, done.stdout + done.stderr assert [order for (order,) in query(project, 'SELECT order_id FROM "RAW".shop__t')] == expected_orders answered = api.ahead_answered # type: ignore[attr-defined] - assert answered.index(4) < answered.index(2) and answered.index(6) < answered.index(2), answered + assert answered.index(4) < answered.index(2), answered assert [start for start, _ in api.ahead].count(4) == 2 # type: ignore[attr-defined] assert 2 <= api.most_open <= 3, api.most_open # type: ignore[attr-defined] diff --git a/website/src/content/docs/docs/sources/python.mdx b/website/src/content/docs/docs/sources/python.mdx index 83d7d9384..7ab51068e 100644 --- a/website/src/content/docs/docs/sources/python.mdx +++ b/website/src/content/docs/docs/sources/python.mdx @@ -391,9 +391,11 @@ api.paginate("/invoices", paginator=sl.offset(limit=50, read_ahead=20)) ``` The first page is read alone, so a short first page is checked as without `read_ahead`. After a full first page, -the next `N` pages are requested together, each through the shared `rate_limit` and retries, and their rows load -strictly in page order. The run ends at the first short (or, for `sl.page_number`, empty) page, and the requests -already sent past it are dropped, so an API may see up to `N` requests more than the pages loaded. +the pages after it are requested two at a time, then four, eight and so on up to `N`, the open requests doubling +each time the pages of the last round have loaded. Each goes through the shared `rate_limit` and retries, and their +rows load strictly in page order. The run ends at the first short (or, for `sl.page_number`, empty) page, and the +requests already sent past it are dropped, so a table or cursor window of a few pages sends only a few requests +more than it loads, and a long one soon has `N` open. `sl.cursor`, `sl.next_link` and `sl.link_header` refuse `read_ahead`, as each page's address comes from the page before it. @@ -408,6 +410,9 @@ mock API answering each page of 100 rows in 80 ms, 1,000 pages took: Raise `read_ahead` until the API's own limits, or a `rate_limit` you set for them, become the bound. +Every job of a load shares one pool of connections, so the cursor windows of a backfill reuse the connections +the first window opened instead of each opening its own. + ### A missing select path A response without the `select` path, as opposed to one where it holds an empty list From a5b72d0d7b17a07838952a01cead8c84d2fa298c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:18:32 +0000 Subject: [PATCH 2/4] perf(http): abort pages read ahead when paging fails, keep the runtime until made Pages read ahead are released to finish only when paging ends normally; a failure, a closed receiver or an aborted job aborts them. A read ahead also checks for the stop before it sends. The shared runtime gets at least two worker threads, and a failure to make it is no longer kept. --- .../sqlload/src/http_source/_helpers/fetch.rs | 3 +++ .../sqlload/src/http_source/_helpers/job.rs | 27 ++++++++++--------- .../sqlload/src/http_source/_helpers/oauth.rs | 4 +-- .../sqlload/src/http_source/_helpers/pages.rs | 23 +++++++++++----- packages/sqlload/tests/test_http_jobs.py | 2 +- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/crates/sqlload/src/http_source/_helpers/fetch.rs b/crates/sqlload/src/http_source/_helpers/fetch.rs index 8d49771fd..a6f4dc95b 100644 --- a/crates/sqlload/src/http_source/_helpers/fetch.rs +++ b/crates/sqlload/src/http_source/_helpers/fetch.rs @@ -43,6 +43,9 @@ pub(crate) async fn fetch(calls: &Calls, call: &PageCall) -> Result None, }; + if calls.stop.is_cancelled() { + return Err(STOPPED.to_owned().into()); + } match once(calls, call, access.as_deref(), !renewed).await? { Attempt::Done(fetched) => return Ok(fetched), Attempt::Refused => { diff --git a/crates/sqlload/src/http_source/_helpers/job.rs b/crates/sqlload/src/http_source/_helpers/job.rs index cbafd9f81..27ff7e689 100644 --- a/crates/sqlload/src/http_source/_helpers/job.rs +++ b/crates/sqlload/src/http_source/_helpers/job.rs @@ -608,23 +608,24 @@ struct Jobs { post: reqwest::Client, } -/// The process's [`Jobs`], made by the first job and kept: each job of a table read in cursor windows (one per window) finds the connections the jobs before it opened still in the pool, instead of a new runtime's empty one, so it opens no new connection, nor repeats a TLS handshake, to a host it already reached. +/// The process's [`Jobs`], made by the first job and kept: each job of a table read in cursor windows (one per window) finds the connections the jobs before it opened still in the pool, instead of a new runtime's empty one, so it opens no new connection, nor repeats a TLS handshake, to a host it already reached; a failure to make them is not kept, so the next job tries again. fn jobs() -> Result<&'static Jobs, String> { - static JOBS: std::sync::OnceLock> = std::sync::OnceLock::new(); - JOBS.get_or_init(|| { - let runtime = tokio::runtime::Builder::new_multi_thread() + static JOBS: std::sync::OnceLock = std::sync::OnceLock::new(); + if let Some(jobs) = JOBS.get() { + return Ok(jobs); + } + let threads = std::thread::available_parallelism().map_or(2, |count| count.get().max(2)); + let made = Jobs { + runtime: tokio::runtime::Builder::new_multi_thread() + .worker_threads(threads) .thread_name("sqlload-http") .enable_all() .build() - .map_err(|error| format!("cannot start the sl.http runtime: {error}"))?; - Ok(Jobs { - runtime, - get: job_client(false)?, - post: job_client(true)?, - }) - }) - .as_ref() - .map_err(Clone::clone) + .map_err(|error| format!("cannot start the sl.http runtime: {error}"))?, + get: job_client(false)?, + post: job_client(true)?, + }; + Ok(JOBS.get_or_init(|| made)) } /// Whether any request of `job` sends a `POST`. diff --git a/crates/sqlload/src/http_source/_helpers/oauth.rs b/crates/sqlload/src/http_source/_helpers/oauth.rs index bd7a3c28e..5c55f9db4 100644 --- a/crates/sqlload/src/http_source/_helpers/oauth.rs +++ b/crates/sqlload/src/http_source/_helpers/oauth.rs @@ -24,7 +24,7 @@ use crate::python_sources::errors::ReadError; /// Every token source of this process, by connection and auth key. static SOURCES: OnceLock>>> = OnceLock::new(); -/// The runtime token requests run on, and its HTTP client: one for the process, outliving any job's runtime, so a token request a cancelled page started still finishes and keeps what the server sent, such as a rotated refresh token. +/// The runtime token requests run on, and its HTTP client: one for the process, apart from the jobs' runtime, so a token request a cancelled page started still finishes and keeps what the server sent, such as a rotated refresh token. static RENEWALS: OnceLock> = OnceLock::new(); @@ -192,7 +192,7 @@ impl TokenSource { .map_err(|error| format!("the OAuth2 token request stopped: {error}"))? } - /// A new access token from the token URL, kept in `state` with any rotated refresh token. It runs on the process's token runtime, so a request cut short while it waits, such as a page read ahead past the last one, or the end of its job's runtime, can't drop a refresh token the server has already replaced. + /// A new access token from the token URL, kept in `state` with any rotated refresh token. It runs on the process's token runtime, so a request cut short while it waits, such as a page read ahead past the last one or an aborted job, can't drop a refresh token the server has already replaced. async fn renewed( &self, http: &reqwest::Client, diff --git a/crates/sqlload/src/http_source/_helpers/pages.rs b/crates/sqlload/src/http_source/_helpers/pages.rs index 93dd533a1..af687a2e7 100644 --- a/crates/sqlload/src/http_source/_helpers/pages.rs +++ b/crates/sqlload/src/http_source/_helpers/pages.rs @@ -10,8 +10,8 @@ use std::collections::VecDeque; use reqwest::Url; use serde_json::{Map, Value}; use tokio::sync::mpsc::Sender; -use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; use crate::http_source::_helpers::cells::pivoted_rows; use crate::http_source::_helpers::fetch::fetch; @@ -28,7 +28,7 @@ use crate::http_source::models::{ }; use crate::python_sources::errors::ReadError; -/// Send every page of `request` at `path` (its placeholders filled) to `pages`, with where its rows are; a page whose selected node is null has no rows and is not sent, and one missing it fails, naming what the response holds, unless the request reads a missing node as no rows (`select_missing`). An offset run that would end after a first page shorter than its limit asks for the next offset once, and fails when rows come back, as the API then caps its pages lower than the limit. With `read_ahead`, offset and page-number pages after a full first page are requested that many at a time (fewer under a memory budget: only as many as pages the size of the largest yet fit in a quarter of each table's share), each through the shared rate limit and retries in a task of its own (so it keeps reading while the pages' receiver holds this loop up, and is aborted when the run ends), and handled strictly in page order, so the run still ends at the first short page, dropping the requests past it. Stops quietly once `pages` closes. A next position (cursor, link or count) already requested for this request fails before it is requested again, naming the value and the page that returned it, and a counted page repeating the previous response fails, rather than loop. With `error_path`, an error a page reports there fails the table before any of that page's rows load. +/// Send every page of `request` at `path` (its placeholders filled) to `pages`, with where its rows are; a page whose selected node is null has no rows and is not sent, and one missing it fails, naming what the response holds, unless the request reads a missing node as no rows (`select_missing`). An offset run that would end after a first page shorter than its limit asks for the next offset once, and fails when rows come back, as the API then caps its pages lower than the limit. With `read_ahead`, offset and page-number pages after a full first page are requested two at a time, then four, eight and so on up to that many (fewer under a memory budget: only as many as pages the size of the largest yet fit in a quarter of each table's share), each through the shared rate limit and retries in a task of its own (so it keeps reading while the pages' receiver holds this loop up), and handled strictly in page order, so the run still ends at the first short page, dropping the requests past it. Stops quietly once `pages` closes. A next position (cursor, link or count) already requested for this request fails before it is requested again, naming the value and the page that returned it, and a counted page repeating the previous response fails, rather than loop. With `error_path`, an error a page reports there fails the table before any of that page's rows load. pub(crate) async fn paginate( calls: &Calls, (request, context): (&Request, &RowContext), @@ -49,7 +49,7 @@ pub(crate) async fn paginate( let mut previous: Option = None; let mut names: Option = None; let ahead = read_ahead(&request.paginator); - let mut in_flight: VecDeque> = VecDeque::new(); + let mut in_flight: VecDeque> = VecDeque::new(); let stop = CancellationToken::new(); let _stopping = stop.clone().drop_guard(); let mut queued: u64 = 1; @@ -78,13 +78,13 @@ pub(crate) async fn paginate( stop: stop.clone(), ..calls.clone() }; - in_flight.push_back(tokio::spawn(async move { + in_flight.push_back(AbortOnDropHandle::new(tokio::spawn(async move { let fetched = fetch(&ahead_calls, &ahead_call).await; AheadPage { call: ahead_call, fetched, } - })); + }))); } if grows { ramped = queued; @@ -108,6 +108,7 @@ pub(crate) async fn paginate( } }; let Some(fetched) = fetched else { + release(in_flight); return Ok(()); }; largest = largest.max(fetched.bytes.len() as u64); @@ -185,12 +186,22 @@ pub(crate) async fn paginate( } match next { Some((next, _)) => call = next, - None => return Ok(()), + None => { + release(in_flight); + return Ok(()); + } } } Ok(()) } +/// Let the pages still read ahead when paging ends finish their requests, so their connections go back to the pool, instead of aborting them with the paging, as a failure or a closed `pages` does; the paging's stop token keeps them from waiting for a turn or retrying. +fn release(in_flight: VecDeque>) { + for handle in in_flight { + drop(handle.detach()); + } +} + /// What identifies a request among those a paginator sent: its URL and body. fn seen_key(call: &PageCall) -> (Url, Option) { (call.url.clone(), call.body.as_ref().map(Value::to_string)) diff --git a/packages/sqlload/tests/test_http_jobs.py b/packages/sqlload/tests/test_http_jobs.py index 322e8545b..cf4f8942d 100644 --- a/packages/sqlload/tests/test_http_jobs.py +++ b/packages/sqlload/tests/test_http_jobs.py @@ -928,7 +928,7 @@ def test_given_read_ahead_when_later_pages_answer_first_and_one_is_retried_then_ assert done.returncode == 0, done.stdout + done.stderr assert [order for (order,) in query(project, 'SELECT order_id FROM "RAW".shop__t')] == expected_orders answered = api.ahead_answered # type: ignore[attr-defined] - assert answered.index(4) < answered.index(2), answered + assert max(index for index, start in enumerate(answered) if start == 4) < answered.index(2), answered assert [start for start, _ in api.ahead].count(4) == 2 # type: ignore[attr-defined] assert 2 <= api.most_open <= 3, api.most_open # type: ignore[attr-defined] From bc4c1ea0c96732c1b57a04795e7443ac018d6b7e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:20:21 +0000 Subject: [PATCH 3/4] docs(http): when a page read ahead still finishes --- crates/sqlload/src/http_source/models.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlload/src/http_source/models.rs b/crates/sqlload/src/http_source/models.rs index 551b418e6..83ed04d71 100644 --- a/crates/sqlload/src/http_source/models.rs +++ b/crates/sqlload/src/http_source/models.rs @@ -450,7 +450,7 @@ pub struct Calls { pub full_page_seen: std::sync::Arc, /// The most the pages `read_ahead` requests may hold at once: a quarter of the table's share of the run's memory budget; `None` without a budget. pub source_part: Option, - /// Cancelled once the pages read ahead are no longer wanted: a request already sent still finishes, so its connection goes back to the pool, but no wait for a turn, retry or other attempt starts. + /// Cancelled once the pages read ahead are no longer wanted: after paging ends normally a request already sent still finishes, so its connection goes back to the pool, but no wait for a turn, retry or other attempt starts. pub stop: tokio_util::sync::CancellationToken, } From 449386a4a8d7350221078adc3631e802a38b6302 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:23:26 +0000 Subject: [PATCH 4/4] style(http): format --- crates/sqlload/src/http_source/_helpers/job.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/sqlload/src/http_source/_helpers/job.rs b/crates/sqlload/src/http_source/_helpers/job.rs index 27ff7e689..fb8721ddc 100644 --- a/crates/sqlload/src/http_source/_helpers/job.rs +++ b/crates/sqlload/src/http_source/_helpers/job.rs @@ -584,9 +584,11 @@ pub(crate) fn drive_job( false => jobs.get.clone(), }; let (sender, mut receiver) = channel(PAGES_AHEAD); - let task = jobs - .runtime - .spawn(run_job(job, (connections.clone(), source_part), (http, sender))); + let task = jobs.runtime.spawn(run_job( + job, + (connections.clone(), source_part), + (http, sender), + )); while let Some(page) = receiver.blocking_recv() { if !deliver(page) { task.abort();