Skip to content

Commit f9b8697

Browse files
authored
perf(location): read warehouse history through one warm reader per sync (#1431)
A code location's warehouse history sync started a new interpreter, and a new warehouse login, for every 100-row page of every state table. One warm runtime interpreter now serves a whole synchronization and is closed when it completes, when project inputs change, or after a failed delivery. Remote warehouses keep one read-only session for the sync and stream each state-table scan through a bounded ordered SELECT, resuming from the keyset cursor when the window ends. Local DuckDB and SQLite files reconnect for every page so their lock never blocks other writers.
1 parent dc21de3 commit f9b8697

10 files changed

Lines changed: 550 additions & 249 deletions

File tree

‎crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs‎

Lines changed: 153 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ use sqlbuild_orchestration::webhooks::main::{
4646
connect_webhook::connect_webhook, run_webhook::run_webhook,
4747
};
4848
use sqlbuild_orchestration::webhooks::models::{Webhook, WebhookSource};
49-
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
49+
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
5050
use tokio::process::{Child, Command};
5151
use tokio::sync::{mpsc, oneshot, watch};
5252
use tokio::task::JoinSet;
@@ -72,8 +72,9 @@ use crate::native_commands::_helpers::terminal::{
7272
};
7373
use crate::native_commands::constants::{
7474
ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_OVERLAP_SECONDS,
75-
METADATA_POLL_SECONDS, METADATA_RECONCILE_SECONDS, METADATA_SYNC_CHECK_SECONDS,
76-
SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND,
75+
METADATA_POLL_SECONDS, METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS,
76+
METADATA_REPLY_BYTES, METADATA_SYNC_CHECK_SECONDS, SCHEDULER_COMMAND,
77+
UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND,
7778
};
7879
use crate::native_commands::errors::PythonNeeded;
7980
use crate::native_commands::main::_run_admin::run_admin;
@@ -358,6 +359,7 @@ pub(crate) fn metadata_sync(
358359
scan_token: None,
359360
scan_delivered: false,
360361
mirror_holds_rows: false,
362+
reader: None,
361363
}
362364
}
363365

@@ -1109,3 +1111,151 @@ pub(crate) fn run_command(
11091111
pub(crate) fn plan_failure(message: &str) -> serde_json::Value {
11101112
json!({"error": {"code": "plan_failed", "message": message, "help": null}})
11111113
}
1114+
1115+
pub(crate) fn open_location_runtime(
1116+
options: &LocationServe,
1117+
script: &Path,
1118+
python_sources: Vec<sqlbuild_cache::project_snapshot::models::PathStamp>,
1119+
) -> Result<LocationCompiler, ControlError> {
1120+
let mut child = location_runtime_child(options, script)?;
1121+
let stdin = child
1122+
.stdin
1123+
.take()
1124+
.ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?;
1125+
let stdout = child.stdout.take().ok_or_else(|| {
1126+
ControlError::Invalid("location interpreter output is unavailable".into())
1127+
})?;
1128+
Ok(LocationCompiler {
1129+
child,
1130+
stdin,
1131+
stdout: BufReader::new(stdout),
1132+
python_sources,
1133+
})
1134+
}
1135+
1136+
pub(crate) async fn stop_location_runtime(
1137+
runtime: Option<LocationCompiler>,
1138+
) -> Result<(), ControlError> {
1139+
if let Some(mut runtime) = runtime {
1140+
if runtime.child.try_wait()?.is_none() {
1141+
runtime.child.kill().await?;
1142+
} else {
1143+
runtime.child.wait().await?;
1144+
}
1145+
}
1146+
Ok(())
1147+
}
1148+
1149+
/// An outer absence means shutdown; an inner absence is a failed read, never an empty table.
1150+
pub(crate) async fn read_metadata_page(
1151+
(options, script): (&LocationServe, &Path),
1152+
mut state: LocationMetadataSync,
1153+
stopped: impl Future<Output = Result<(), ControlError>>,
1154+
) -> Result<Option<(LocationMetadataSync, Option<WarehouseHistoryPage>)>, ControlError> {
1155+
if let Some(statistics) = &state.statistics
1156+
&& statistics.names.is_empty()
1157+
{
1158+
let page = WarehouseHistoryPage {
1159+
records: Vec::new(),
1160+
next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]),
1161+
change_token: None,
1162+
unchanged: false,
1163+
};
1164+
return Ok(Some((state, Some(page))));
1165+
}
1166+
let request = history_request(&state)?;
1167+
let reader = match state.reader.take() {
1168+
Some(reader) => reader,
1169+
None => match open_location_runtime(options, script, Vec::new()) {
1170+
Ok(reader) => reader,
1171+
Err(_) => return Ok(Some((state, None))),
1172+
},
1173+
};
1174+
tokio::pin!(stopped);
1175+
tokio::select! {
1176+
result = &mut stopped => {
1177+
result?;
1178+
Ok(None)
1179+
},
1180+
result = tokio::time::timeout(
1181+
Duration::from_secs(METADATA_POLL_SECONDS),
1182+
exchange_metadata_read(reader, request),
1183+
) => match result {
1184+
Ok((reader, Ok(page))) => {
1185+
state.reader = Some(reader);
1186+
Ok(Some((state, page)))
1187+
}
1188+
Ok((reader, Err(_))) => {
1189+
stop_location_runtime(Some(reader)).await?;
1190+
Ok(Some((state, None)))
1191+
}
1192+
Err(_) => Ok(Some((state, None))),
1193+
},
1194+
}
1195+
}
1196+
1197+
/// A completed synchronization closes its reader, so no warehouse session outlives the sync.
1198+
pub(crate) async fn finish_metadata_page(
1199+
state: LocationMetadataSync,
1200+
) -> Result<Option<LocationMetadataSync>, ControlError> {
1201+
if state.idle {
1202+
return Ok(Some(close_metadata_reader(state).await?));
1203+
}
1204+
Ok(Some(state))
1205+
}
1206+
1207+
/// End of input lets the reader close its warehouse session; one that ignores it is killed.
1208+
pub(crate) async fn close_metadata_reader(
1209+
mut state: LocationMetadataSync,
1210+
) -> Result<LocationMetadataSync, ControlError> {
1211+
if let Some(LocationCompiler {
1212+
mut child, stdin, ..
1213+
}) = state.reader.take()
1214+
{
1215+
drop(stdin);
1216+
match tokio::time::timeout(
1217+
Duration::from_secs(METADATA_READER_CLOSE_SECONDS),
1218+
child.wait(),
1219+
)
1220+
.await
1221+
{
1222+
Ok(waited) => drop(waited?),
1223+
Err(_) => child.kill().await?,
1224+
}
1225+
}
1226+
Ok(state)
1227+
}
1228+
1229+
/// A reply of `null` means the reader's read failed and it dropped its warehouse session.
1230+
async fn exchange_metadata_read(
1231+
mut reader: LocationCompiler,
1232+
request: Vec<u8>,
1233+
) -> (
1234+
LocationCompiler,
1235+
Result<Option<WarehouseHistoryPage>, ControlError>,
1236+
) {
1237+
let result = async {
1238+
reader.stdin.write_all(&request).await?;
1239+
reader.stdin.write_all(b"\n").await?;
1240+
reader.stdin.flush().await?;
1241+
let mut line = String::new();
1242+
let read = (&mut reader.stdout)
1243+
.take(METADATA_REPLY_BYTES + 1)
1244+
.read_line(&mut line)
1245+
.await?;
1246+
if read == 0 {
1247+
return Err(ControlError::Invalid(
1248+
"location metadata reader exited before replying".into(),
1249+
));
1250+
}
1251+
if read as u64 > METADATA_REPLY_BYTES || !line.ends_with('\n') {
1252+
return Err(ControlError::Invalid(
1253+
"location reply exceeds its byte limit".into(),
1254+
));
1255+
}
1256+
serde_json::from_str(&line)
1257+
.map_err(|_| ControlError::Invalid("invalid location reply".into()))
1258+
}
1259+
.await;
1260+
(reader, result)
1261+
}

‎crates/sqlbuild-cli/src/native_commands/constants.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30;
9595
pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576;
9696
/// Read timeout, and the delay before retrying a failed page of an unfinished synchronization.
9797
pub(crate) const METADATA_POLL_SECONDS: u64 = 30;
98+
/// A metadata reader that ignores end of input this long after a synchronization is killed.
99+
pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5;
98100
/// How often an idle location asks the server, never the warehouse, whether a run ended.
99101
pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5;
100102
/// Full key-set reconciliation is independent of the incremental poll and overlap window.

‎crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs‎

Lines changed: 10 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,10 @@
33
use std::path::Path;
44

55
use sqlbuild_orchestration::store::errors::ControlError;
6-
use tokio::io::BufReader;
76
use tokio::sync::mpsc;
87

9-
use crate::native_commands::_helpers::execution::{
10-
exchange_location_generation, location_runtime_child,
11-
};
8+
use crate::native_commands::_helpers::execution::exchange_location_generation;
9+
use crate::native_commands::_helpers::execution::{open_location_runtime, stop_location_runtime};
1210
use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe};
1311

1412
pub(crate) async fn run_location_compiler(
@@ -23,10 +21,14 @@ pub(crate) async fn run_location_compiler(
2321
.as_ref()
2422
.is_some_and(|compiler| compiler.python_sources != request.python_sources)
2523
{
26-
stop_compiler(compiler.take()).await?;
24+
stop_location_runtime(compiler.take()).await?;
2725
}
2826
if compiler.is_none() {
29-
compiler = Some(open_compiler(options, script, request.python_sources)?);
27+
compiler = Some(open_location_runtime(
28+
options,
29+
script,
30+
request.python_sources,
31+
)?);
3032
}
3133
let active = compiler
3234
.as_mut()
@@ -40,44 +42,12 @@ pub(crate) async fn run_location_compiler(
4042
.reply
4143
.send(result.map_err(|error| error.to_string()));
4244
if failed {
43-
stop_compiler(compiler.take()).await?;
45+
stop_location_runtime(compiler.take()).await?;
4446
}
4547
}
4648
Ok(())
4749
}
4850
.await;
49-
stop_compiler(compiler).await?;
51+
stop_location_runtime(compiler).await?;
5052
result
5153
}
52-
53-
fn open_compiler(
54-
options: &LocationServe,
55-
script: &Path,
56-
python_sources: Vec<sqlbuild_cache::project_snapshot::models::PathStamp>,
57-
) -> Result<LocationCompiler, ControlError> {
58-
let mut child = location_runtime_child(options, script)?;
59-
let stdin = child
60-
.stdin
61-
.take()
62-
.ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?;
63-
let stdout = child.stdout.take().ok_or_else(|| {
64-
ControlError::Invalid("location interpreter output is unavailable".into())
65-
})?;
66-
Ok(LocationCompiler {
67-
child,
68-
stdin,
69-
stdout: BufReader::new(stdout),
70-
python_sources,
71-
})
72-
}
73-
74-
async fn stop_compiler(compiler: Option<LocationCompiler>) -> Result<(), ControlError> {
75-
if let Some(mut compiler) = compiler {
76-
if compiler.child.try_wait()?.is_none() {
77-
compiler.child.kill().await?;
78-
} else {
79-
compiler.child.wait().await?;
80-
}
81-
}
82-
Ok(())
83-
}

‎crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs‎

Lines changed: 11 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,18 @@ use sqlbuild_orchestration::platform::models::{
99
LocationWorkClient, LocationWorkPoll, WarehouseSyncFailure,
1010
};
1111
use sqlbuild_orchestration::store::errors::ControlError;
12-
use sqlbuild_orchestration::warehouse_metadata::models::{
13-
WarehouseHistoryKind, WarehouseHistoryPage,
14-
};
12+
use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryKind;
1513
use tokio::time::Instant;
1614

1715
use crate::argv::models::Stream;
1816
use crate::native_commands::_helpers::execution::{
19-
acknowledge_metadata_page, begin_after_run, freeze_history_page, history_request,
20-
load_watermark, location_result, metadata_health, next_scan, statistics_page,
17+
acknowledge_metadata_page, begin_after_run, freeze_history_page, load_watermark,
18+
metadata_health, next_scan, statistics_page,
2119
};
20+
use crate::native_commands::_helpers::execution::{finish_metadata_page, read_metadata_page};
2221
use crate::native_commands::_helpers::paths::location_inputs;
2322
use crate::native_commands::_helpers::terminal::write_output;
24-
use crate::native_commands::constants::{METADATA_POLL_SECONDS, METADATA_REPLY_BYTES};
23+
use crate::native_commands::constants::METADATA_POLL_SECONDS;
2524
use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch};
2625

2726
/// One bounded source or mirror page leaves named work polling a turn between pages.
@@ -72,10 +71,12 @@ pub(super) async fn sync_location_metadata(
7271
return retry_metadata(state, error);
7372
}
7473
}
75-
let Some(page) = read_metadata_page((options, script), &state, stopped.as_mut()).await?
74+
let Some((read, page)) =
75+
read_metadata_page((options, script), state, stopped.as_mut()).await?
7676
else {
7777
return Ok(None);
7878
};
79+
state = read;
7980
let Some(page) = page else {
8081
tokio::select! {
8182
result = &mut stopped => { result?; return Ok(None); },
@@ -97,7 +98,7 @@ pub(super) async fn sync_location_metadata(
9798
result = &mut stopped => { result?; return Ok(None); },
9899
() = metadata_health(&state, (client, poll), (started, None)) => {},
99100
}
100-
return Ok(Some(next_scan(state)));
101+
return finish_metadata_page(next_scan(state)).await;
101102
}
102103
state = freeze_history_page(state, page, poll)?;
103104
}
@@ -119,7 +120,7 @@ pub(super) async fn sync_location_metadata(
119120
result = &mut stopped => { result?; return Ok(None); },
120121
() = metadata_health(&state, (client, poll), (started, None)) => {},
121122
}
122-
Ok(Some(acknowledge_metadata_page(state)))
123+
finish_metadata_page(acknowledge_metadata_page(state)).await
123124
}
124125

125126
fn retry_metadata(
@@ -132,6 +133,7 @@ fn retry_metadata(
132133
return Err(ControlError::Invalid(error.to_string()));
133134
}
134135
state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS);
136+
state.reader = None;
135137
drop(write_output(&[(
136138
Stream::Stderr,
137139
&format!(
@@ -140,47 +142,3 @@ fn retry_metadata(
140142
)]));
141143
Ok(Some(state))
142144
}
143-
144-
/// An outer absence means shutdown; an inner absence is a failed read, never an empty table.
145-
async fn read_metadata_page(
146-
(options, script): (&LocationServe, &Path),
147-
state: &LocationMetadataSync,
148-
stopped: impl Future<Output = Result<(), ControlError>>,
149-
) -> Result<Option<Option<WarehouseHistoryPage>>, ControlError> {
150-
if let Some(statistics) = &state.statistics
151-
&& statistics.names.is_empty()
152-
{
153-
return Ok(Some(Some(WarehouseHistoryPage {
154-
records: Vec::new(),
155-
next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]),
156-
change_token: None,
157-
unchanged: false,
158-
})));
159-
}
160-
let (mut child, reply) = match location_result::<Option<WarehouseHistoryPage>>(
161-
(options, script),
162-
(history_request(state)?, METADATA_REPLY_BYTES),
163-
) {
164-
Ok(exchange) => exchange,
165-
Err(_) => return Ok(Some(None)),
166-
};
167-
tokio::pin!(stopped);
168-
let outcome = tokio::select! {
169-
result = &mut stopped => { child.kill().await?; result?; return Ok(None); },
170-
result = tokio::time::timeout(Duration::from_secs(METADATA_POLL_SECONDS), async {
171-
let page = reply.await?;
172-
if !child.wait().await?.success() {
173-
return Err(ControlError::Invalid("location metadata interpreter failed".into()));
174-
}
175-
Ok(page)
176-
}) => result,
177-
};
178-
if child.try_wait()?.is_none() {
179-
child.kill().await?;
180-
}
181-
let page = match outcome {
182-
Ok(Ok(page)) => page,
183-
Ok(Err(_)) | Err(_) => None,
184-
};
185-
Ok(Some(page))
186-
}

0 commit comments

Comments
 (0)