diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 81706ccd14..d6903fafd9 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -46,7 +46,7 @@ use sqlbuild_orchestration::webhooks::main::{ connect_webhook::connect_webhook, run_webhook::run_webhook, }; use sqlbuild_orchestration::webhooks::models::{Webhook, WebhookSource}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::process::{Child, Command}; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinSet; @@ -72,8 +72,9 @@ use crate::native_commands::_helpers::terminal::{ }; use crate::native_commands::constants::{ ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_OVERLAP_SECONDS, - METADATA_POLL_SECONDS, METADATA_RECONCILE_SECONDS, METADATA_SYNC_CHECK_SECONDS, - SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, + METADATA_POLL_SECONDS, METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS, + METADATA_REPLY_BYTES, METADATA_SYNC_CHECK_SECONDS, SCHEDULER_COMMAND, + UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; @@ -358,6 +359,7 @@ pub(crate) fn metadata_sync( scan_token: None, scan_delivered: false, mirror_holds_rows: false, + reader: None, } } @@ -1109,3 +1111,151 @@ pub(crate) fn run_command( pub(crate) fn plan_failure(message: &str) -> serde_json::Value { json!({"error": {"code": "plan_failed", "message": message, "help": null}}) } + +pub(crate) fn open_location_runtime( + options: &LocationServe, + script: &Path, + python_sources: Vec, +) -> Result { + let mut child = location_runtime_child(options, script)?; + let stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?; + let stdout = child.stdout.take().ok_or_else(|| { + ControlError::Invalid("location interpreter output is unavailable".into()) + })?; + Ok(LocationCompiler { + child, + stdin, + stdout: BufReader::new(stdout), + python_sources, + }) +} + +pub(crate) async fn stop_location_runtime( + runtime: Option, +) -> Result<(), ControlError> { + if let Some(mut runtime) = runtime { + if runtime.child.try_wait()?.is_none() { + runtime.child.kill().await?; + } else { + runtime.child.wait().await?; + } + } + Ok(()) +} + +/// An outer absence means shutdown; an inner absence is a failed read, never an empty table. +pub(crate) async fn read_metadata_page( + (options, script): (&LocationServe, &Path), + mut state: LocationMetadataSync, + stopped: impl Future>, +) -> Result)>, ControlError> { + if let Some(statistics) = &state.statistics + && statistics.names.is_empty() + { + let page = WarehouseHistoryPage { + records: Vec::new(), + next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]), + change_token: None, + unchanged: false, + }; + return Ok(Some((state, Some(page)))); + } + let request = history_request(&state)?; + let reader = match state.reader.take() { + Some(reader) => reader, + None => match open_location_runtime(options, script, Vec::new()) { + Ok(reader) => reader, + Err(_) => return Ok(Some((state, None))), + }, + }; + tokio::pin!(stopped); + tokio::select! { + result = &mut stopped => { + result?; + Ok(None) + }, + result = tokio::time::timeout( + Duration::from_secs(METADATA_POLL_SECONDS), + exchange_metadata_read(reader, request), + ) => match result { + Ok((reader, Ok(page))) => { + state.reader = Some(reader); + Ok(Some((state, page))) + } + Ok((reader, Err(_))) => { + stop_location_runtime(Some(reader)).await?; + Ok(Some((state, None))) + } + Err(_) => Ok(Some((state, None))), + }, + } +} + +/// A completed synchronization closes its reader, so no warehouse session outlives the sync. +pub(crate) async fn finish_metadata_page( + state: LocationMetadataSync, +) -> Result, ControlError> { + if state.idle { + return Ok(Some(close_metadata_reader(state).await?)); + } + Ok(Some(state)) +} + +/// End of input lets the reader close its warehouse session; one that ignores it is killed. +pub(crate) async fn close_metadata_reader( + mut state: LocationMetadataSync, +) -> Result { + if let Some(LocationCompiler { + mut child, stdin, .. + }) = state.reader.take() + { + drop(stdin); + match tokio::time::timeout( + Duration::from_secs(METADATA_READER_CLOSE_SECONDS), + child.wait(), + ) + .await + { + Ok(waited) => drop(waited?), + Err(_) => child.kill().await?, + } + } + Ok(state) +} + +/// A reply of `null` means the reader's read failed and it dropped its warehouse session. +async fn exchange_metadata_read( + mut reader: LocationCompiler, + request: Vec, +) -> ( + LocationCompiler, + Result, ControlError>, +) { + let result = async { + reader.stdin.write_all(&request).await?; + reader.stdin.write_all(b"\n").await?; + reader.stdin.flush().await?; + let mut line = String::new(); + let read = (&mut reader.stdout) + .take(METADATA_REPLY_BYTES + 1) + .read_line(&mut line) + .await?; + if read == 0 { + return Err(ControlError::Invalid( + "location metadata reader exited before replying".into(), + )); + } + if read as u64 > METADATA_REPLY_BYTES || !line.ends_with('\n') { + return Err(ControlError::Invalid( + "location reply exceeds its byte limit".into(), + )); + } + serde_json::from_str(&line) + .map_err(|_| ControlError::Invalid("invalid location reply".into())) + } + .await; + (reader, result) +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index ab853320e3..f1b8a95848 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -95,6 +95,8 @@ pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; /// Read timeout, and the delay before retrying a failed page of an unfinished synchronization. pub(crate) const METADATA_POLL_SECONDS: u64 = 30; +/// A metadata reader that ignores end of input this long after a synchronization is killed. +pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5; /// How often an idle location asks the server, never the warehouse, whether a run ended. pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5; /// Full key-set reconciliation is independent of the incremental poll and overlap window. diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs index 2a20ef09b2..e3035ec0e9 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs @@ -3,12 +3,10 @@ use std::path::Path; use sqlbuild_orchestration::store::errors::ControlError; -use tokio::io::BufReader; use tokio::sync::mpsc; -use crate::native_commands::_helpers::execution::{ - exchange_location_generation, location_runtime_child, -}; +use crate::native_commands::_helpers::execution::exchange_location_generation; +use crate::native_commands::_helpers::execution::{open_location_runtime, stop_location_runtime}; use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe}; pub(crate) async fn run_location_compiler( @@ -23,10 +21,14 @@ pub(crate) async fn run_location_compiler( .as_ref() .is_some_and(|compiler| compiler.python_sources != request.python_sources) { - stop_compiler(compiler.take()).await?; + stop_location_runtime(compiler.take()).await?; } if compiler.is_none() { - compiler = Some(open_compiler(options, script, request.python_sources)?); + compiler = Some(open_location_runtime( + options, + script, + request.python_sources, + )?); } let active = compiler .as_mut() @@ -40,44 +42,12 @@ pub(crate) async fn run_location_compiler( .reply .send(result.map_err(|error| error.to_string())); if failed { - stop_compiler(compiler.take()).await?; + stop_location_runtime(compiler.take()).await?; } } Ok(()) } .await; - stop_compiler(compiler).await?; + stop_location_runtime(compiler).await?; result } - -fn open_compiler( - options: &LocationServe, - script: &Path, - python_sources: Vec, -) -> Result { - let mut child = location_runtime_child(options, script)?; - let stdin = child - .stdin - .take() - .ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?; - let stdout = child.stdout.take().ok_or_else(|| { - ControlError::Invalid("location interpreter output is unavailable".into()) - })?; - Ok(LocationCompiler { - child, - stdin, - stdout: BufReader::new(stdout), - python_sources, - }) -} - -async fn stop_compiler(compiler: Option) -> Result<(), ControlError> { - if let Some(mut compiler) = compiler { - if compiler.child.try_wait()?.is_none() { - compiler.child.kill().await?; - } else { - compiler.child.wait().await?; - } - } - Ok(()) -} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index 4e90fadcd5..ea5ddaf6f2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -9,19 +9,18 @@ use sqlbuild_orchestration::platform::models::{ LocationWorkClient, LocationWorkPoll, WarehouseSyncFailure, }; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryKind, WarehouseHistoryPage, -}; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryKind; use tokio::time::Instant; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - acknowledge_metadata_page, begin_after_run, freeze_history_page, history_request, - load_watermark, location_result, metadata_health, next_scan, statistics_page, + acknowledge_metadata_page, begin_after_run, freeze_history_page, load_watermark, + metadata_health, next_scan, statistics_page, }; +use crate::native_commands::_helpers::execution::{finish_metadata_page, read_metadata_page}; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; -use crate::native_commands::constants::{METADATA_POLL_SECONDS, METADATA_REPLY_BYTES}; +use crate::native_commands::constants::METADATA_POLL_SECONDS; use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; /// One bounded source or mirror page leaves named work polling a turn between pages. @@ -72,10 +71,12 @@ pub(super) async fn sync_location_metadata( return retry_metadata(state, error); } } - let Some(page) = read_metadata_page((options, script), &state, stopped.as_mut()).await? + let Some((read, page)) = + read_metadata_page((options, script), state, stopped.as_mut()).await? else { return Ok(None); }; + state = read; let Some(page) = page else { tokio::select! { result = &mut stopped => { result?; return Ok(None); }, @@ -97,7 +98,7 @@ pub(super) async fn sync_location_metadata( result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, } - return Ok(Some(next_scan(state))); + return finish_metadata_page(next_scan(state)).await; } state = freeze_history_page(state, page, poll)?; } @@ -119,7 +120,7 @@ pub(super) async fn sync_location_metadata( result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, } - Ok(Some(acknowledge_metadata_page(state))) + finish_metadata_page(acknowledge_metadata_page(state)).await } fn retry_metadata( @@ -132,6 +133,7 @@ fn retry_metadata( return Err(ControlError::Invalid(error.to_string())); } state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + state.reader = None; drop(write_output(&[( Stream::Stderr, &format!( @@ -140,47 +142,3 @@ fn retry_metadata( )])); Ok(Some(state)) } - -/// An outer absence means shutdown; an inner absence is a failed read, never an empty table. -async fn read_metadata_page( - (options, script): (&LocationServe, &Path), - state: &LocationMetadataSync, - stopped: impl Future>, -) -> Result>, ControlError> { - if let Some(statistics) = &state.statistics - && statistics.names.is_empty() - { - return Ok(Some(Some(WarehouseHistoryPage { - records: Vec::new(), - next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]), - change_token: None, - unchanged: false, - }))); - } - let (mut child, reply) = match location_result::>( - (options, script), - (history_request(state)?, METADATA_REPLY_BYTES), - ) { - Ok(exchange) => exchange, - Err(_) => return Ok(Some(None)), - }; - tokio::pin!(stopped); - let outcome = tokio::select! { - result = &mut stopped => { child.kill().await?; result?; return Ok(None); }, - result = tokio::time::timeout(Duration::from_secs(METADATA_POLL_SECONDS), async { - let page = reply.await?; - if !child.wait().await?.success() { - return Err(ControlError::Invalid("location metadata interpreter failed".into())); - } - Ok(page) - }) => result, - }; - if child.try_wait()?.is_none() { - child.kill().await?; - } - let page = match outcome { - Ok(Ok(page)) => page, - Ok(Err(_)) | Err(_) => None, - }; - Ok(Some(page)) -} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 9968de6124..b97562018e 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -15,8 +15,8 @@ use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - check_published_location, compile_generation, location_generation, location_heartbeats, - metadata_sync, report_location_health, shutdown_signal, + check_published_location, close_metadata_reader, compile_generation, location_generation, + location_heartbeats, metadata_sync, report_location_health, shutdown_signal, }; use crate::native_commands::_helpers::paths::{ exclude_location_databases, location_watch, observe_location_inputs, @@ -93,6 +93,9 @@ async fn watch_location( continue; } published_generation = None; + if let Some(state) = metadata.take() { + metadata = Some(close_metadata_reader(state).await?); + } let generation = tokio::select! { result = &mut stopped => return result, generation = location_generation(options, bearer) => generation?, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index d1911db3b1..4123e2b9e1 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -170,7 +170,7 @@ pub(crate) struct LocationCompileRequest { pub(crate) reply: tokio::sync::oneshot::Sender, String>>, } -/// Only this task owns the isolated interpreter and its private stdio protocol. +/// Only its owner (the compiler task or one metadata synchronization) uses this interpreter's stdio. #[derive(Debug)] pub(crate) struct LocationCompiler { pub(crate) child: tokio::process::Child, @@ -217,6 +217,8 @@ pub(crate) struct LocationMetadataSync { pub(crate) scan_delivered: bool, /// The server already mirrors rows of the current scan's table, per its watermark. pub(crate) mirror_holds_rows: bool, + /// One warm runtime interpreter and warehouse session serve every page of a synchronization. + pub(crate) reader: Option, } /// A table's change token after a complete read, and what that read proves about the mirror. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 97e23fcf6f..5df26a40db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -1,4 +1,4 @@ -//! A location keeps its interpreter warm; only the supervisor writes generation requests. +//! A location keeps its interpreter, and a synchronization its warehouse session, warm. use std::io::{self, Write}; @@ -14,93 +14,132 @@ use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; -use crate::bindings::_helpers::native_host::warehouse_metadata::read_history; +use crate::bindings::_helpers::native_host::warehouse_metadata::{ + close_history_session, open_history_session, read_history, +}; use crate::bindings::_helpers::recall::console::write_stderr; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; let redirect = redirect_stdout_to_stderr(py)?; within(&redirect, |_| { - loop { - let line = py.detach(|| -> io::Result> { - let mut line = String::new(); - Ok((io::stdin().read_line(&mut line)? != 0).then_some(line)) - })?; - let Some(line) = line else { - break; - }; - let request: serde_json::Value = serde_json::from_str(&line).map_err(|_| { - PyValueError::new_err( - "runtime expects a JSON generation or sensor evaluation per line", - ) - })?; - let output = if let Some(generation) = request.as_i64() { - options.set_item("generation", generation)?; - let arguments = location_arguments(py, Some(options.clone()))?; - let record = match publish_location(py, &arguments) { - Ok(record) => Some(record), - Err(error) => { - write_stderr( - py, - &format!("Location compilation/publication failed: {error}\n"), - )?; - None - } + let mut session = None; + let mut stream = None; + let mut serve = || -> PyResult { + loop { + let line = py.detach(|| -> io::Result> { + let mut line = String::new(); + Ok((io::stdin().read_line(&mut line)? != 0).then_some(line)) + })?; + let Some(line) = line else { + break Ok(0); }; - serde_json::to_string(&record) - } else if WarehouseHistoryKind::ALL - .iter() - .any(|kind| request.get(kind.as_str()).is_some()) - { - let metadata: WarehouseHistoryRead = - serde_json::from_value(request).map_err(|_| { - PyValueError::new_err("runtime received an invalid metadata read") - })?; - let arguments = location_arguments(py, Some(options.clone()))?; - let page = match read_history(py, &arguments, &metadata) { - Ok(page) => Some(page), - Err(_) => { + let request: serde_json::Value = serde_json::from_str(&line).map_err(|_| { + PyValueError::new_err( + "runtime expects a JSON generation or sensor evaluation per line", + ) + })?; + let output = if let Some(generation) = request.as_i64() { + options.set_item("generation", generation)?; + let arguments = location_arguments(py, Some(options.clone()))?; + let record = match publish_location(py, &arguments) { + Ok(record) => Some(record), + Err(error) => { + write_stderr( + py, + &format!("Location compilation/publication failed: {error}\n"), + )?; + None + } + }; + serde_json::to_string(&record) + } else if WarehouseHistoryKind::ALL + .iter() + .any(|kind| request.get(kind.as_str()).is_some()) + { + let metadata: WarehouseHistoryRead = + serde_json::from_value(request).map_err(|_| { + PyValueError::new_err("runtime received an invalid metadata read") + })?; + if session.is_none() { + let arguments = location_arguments(py, Some(options.clone()))?; + session = match open_history_session(py, &arguments) { + Ok(opened) => Some(opened), + Err(_) => { + write_stderr( + py, + "Warehouse connection for metadata reads failed.\n", + )?; + None + } + }; + } + let page = match &session { + Some(opened) => { + match read_history(py, (opened, stream.take()), &metadata) { + Ok((page, open)) => { + stream = open; + Some(page) + } + Err(_) => None, + } + } + None => None, + }; + if session.as_ref().is_some_and(|opened| opened.local) + && let Some(opened) = session.take() + { + drop(close_history_session(opened, stream.take())); + } + if page.is_none() { + if let Some(opened) = session.take() { + drop(close_history_session(opened, None)); + } write_stderr(py, "Warehouse metadata read failed. No page delivered.\n")?; - None } - }; - serde_json::to_string(&page) - } else { - let evaluation: SensorEvaluationRequest = - serde_json::from_value(request).map_err(|_| { - PyValueError::new_err("runtime received an invalid sensor evaluation") - })?; - let arguments = location_arguments(py, Some(options.clone()))?; - if arguments.value("location_id")?.extract::()? - != evaluation.location_id.to_string() - || arguments.value("selected_target")?.extract::()? != evaluation.target - { - return Err(PyValueError::new_err( - "sensor work belongs to another location or target", - )); - } - let directory = arguments - .value("project_dir")? - .extract::()?; - let outcome = match evaluate_sensor(py, &directory, &evaluation) { - Ok(outcome) => outcome, - Err(_) => { - write_stderr( - py, - "Custom sensor evaluation failed. No cursor or run requests committed.\n", - )?; - CustomSensorOutcome::Failed + serde_json::to_string(&page) + } else { + let evaluation: SensorEvaluationRequest = serde_json::from_value(request) + .map_err(|_| { + PyValueError::new_err("runtime received an invalid sensor evaluation") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + if arguments.value("location_id")?.extract::()? + != evaluation.location_id.to_string() + || arguments.value("selected_target")?.extract::()? + != evaluation.target + { + return Err(PyValueError::new_err( + "sensor work belongs to another location or target", + )); } + let directory = arguments + .value("project_dir")? + .extract::()?; + let outcome = match evaluate_sensor(py, &directory, &evaluation) { + Ok(outcome) => outcome, + Err(_) => { + write_stderr( + py, + "Custom sensor evaluation failed. No cursor or run requests committed.\n", + )?; + CustomSensorOutcome::Failed + } + }; + serde_json::to_string(&outcome) }; - serde_json::to_string(&outcome) - }; - let output = output.map_err(|error| PyValueError::new_err(error.to_string()))?; - py.detach(|| -> io::Result<()> { - let mut stdout = io::stdout().lock(); - writeln!(stdout, "{output}")?; - stdout.flush() - })?; + let output = output.map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| -> io::Result<()> { + let mut stdout = io::stdout().lock(); + writeln!(stdout, "{output}")?; + stdout.flush() + })?; + } + }; + let result = serve(); + if let Some(opened) = session.take() { + close_history_session(opened, stream.take())?; } - Ok(0) + result }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 52957406f1..84742b4d46 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -21,3 +21,6 @@ pub(crate) const JOB_LABEL: &str = "job"; pub(crate) const GROUP_LABEL: &str = "job group"; pub(crate) const SENSOR_LABEL: &str = "sensor"; pub(crate) const CALENDAR_LABEL: &str = "calendar schedule"; + +/// Pages one bounded state-history SELECT streams before the scan continues from its cursor. +pub(crate) const HISTORY_STREAM_PAGES: usize = 50; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 49e7e71160..ec29ddfbb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -36,6 +36,7 @@ use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; use crate::bindings::_helpers::freshness::state_io::row_record; use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; +use crate::bindings::_helpers::native_host::constants::HISTORY_STREAM_PAGES; use crate::bindings::_helpers::native_host::warehouse_catalog::read_catalog; use crate::bindings::_helpers::native_host::warehouse_statistics::read_statistics; use crate::bindings::_helpers::query_command::project::{ @@ -44,30 +45,35 @@ use crate::bindings::_helpers::query_command::project::{ use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::{construct::record_class, ids::RecordId}; -pub(crate) fn read_history( - py: Python<'_>, - arguments: &KeywordArguments<'_>, - request: &WarehouseHistoryRead, -) -> PyResult { - let location = match request { - WarehouseHistoryRead::Statistics(read) => &read.location, - WarehouseHistoryRead::Catalog(read) => &read.location, - WarehouseHistoryRead::SourceFreshness(read) => &read.location, - WarehouseHistoryRead::Fingerprints(read) => &read.location, - WarehouseHistoryRead::AuditResults(read) - | WarehouseHistoryRead::JanitorEvents(read) - | WarehouseHistoryRead::Migrations(read) - | WarehouseHistoryRead::OldNameViews(read) - | WarehouseHistoryRead::Microbatches(read) - | WarehouseHistoryRead::ColumnMigrations(read) => &read.location, - WarehouseHistoryRead::NodeResults(read) => &read.location, - WarehouseHistoryRead::Consumption(read) => &read.location, - }; - if location.schema.trim().is_empty() { - return Err(PyValueError::new_err( - "metadata state schema must not be blank", - )); - } +/// One read-only adapter session serves every page of one synchronization. +pub(crate) struct HistorySession<'py> { + adapter: Bound<'py, PyAny>, + connection: Bound<'py, PyAny>, + /// A local database file, whose read-only connection would block other processes' writers. + pub(crate) local: bool, +} + +/// One ordered SELECT per table scan; each page fetches the next rows of the same result. +pub(crate) struct HistoryStream<'py> { + /// The request that opened the stream, without its cursor, limit or token. + scan: Value, + /// A DB-API cursor, or DuckDB's connection; dropped, never closed, when discarded. + cursor: Bound<'py, PyAny>, + /// The cursor the last delivered page ended at; only a request for exactly it continues. + next_after: Vec>, + /// Rows the bounded SELECT can still return; at zero the next page reads from its cursor. + remaining: usize, + qualified: String, + columns: Vec, + timestamp: String, + cursor_indexes: Vec, + since: Option>, +} + +pub(crate) fn open_history_session<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, +) -> PyResult> { let ProjectAdapter { project_dir, discovered, @@ -84,10 +90,10 @@ pub(crate) fn read_history( true, )?; let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; - if let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS + let local_path_key = LOCAL_DATABASE_PATH_KEYS .iter() - .find(|(name, _)| *name == adapter_name) - { + .find(|(name, _)| *name == adapter_name); + if let Some((_, path_key)) = local_path_key { let config = connection_config.cast::()?; let path: Option = config .get_item(path_key)? @@ -108,18 +114,71 @@ pub(crate) fn read_history( } } let connection = adapter.call_method1("connect", (connection_config,))?; - let read = read_page(py, (&adapter, &connection), request); - let closed = adapter.call_method1("close", (&connection,)); - let page = read?; - closed?; - Ok(page) + Ok(HistorySession { + adapter, + connection, + local: local_path_key.is_some(), + }) } -fn read_page( - py: Python<'_>, - (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +pub(crate) fn close_history_session( + session: HistorySession<'_>, + stream: Option>, +) -> PyResult<()> { + discard_stream(&session.connection, stream)?; + session + .adapter + .call_method1("close", (&session.connection,))?; + Ok(()) +} + +/// Returns the page and the scan's still-open stream, which only its next page continues. +pub(crate) fn read_history<'py>( + py: Python<'py>, + (session, stream): (&HistorySession<'py>, Option>), + request: &WarehouseHistoryRead, +) -> PyResult<(WarehouseHistoryPage, Option>)> { + let location = match request { + WarehouseHistoryRead::Statistics(read) => &read.location, + WarehouseHistoryRead::Catalog(read) => &read.location, + WarehouseHistoryRead::SourceFreshness(read) => &read.location, + WarehouseHistoryRead::Fingerprints(read) => &read.location, + WarehouseHistoryRead::AuditResults(read) + | WarehouseHistoryRead::JanitorEvents(read) + | WarehouseHistoryRead::Migrations(read) + | WarehouseHistoryRead::OldNameViews(read) + | WarehouseHistoryRead::Microbatches(read) + | WarehouseHistoryRead::ColumnMigrations(read) => &read.location, + WarehouseHistoryRead::NodeResults(read) => &read.location, + WarehouseHistoryRead::Consumption(read) => &read.location, + }; + if location.schema.trim().is_empty() { + return Err(PyValueError::new_err( + "metadata state schema must not be blank", + )); + } + read_page(py, (session, stream), request) +} + +/// Any request but the open scan's next page discards that result before running other SQL. +fn read_page<'py>( + py: Python<'py>, + (session, stream): (&HistorySession<'py>, Option>), request: &WarehouseHistoryRead, -) -> PyResult { +) -> PyResult<(WarehouseHistoryPage, Option>)> { + let (scan, after_value) = scan_identity(request)?; + let continued = match stream { + Some(stream) + if stream.scan == scan && after_value.as_ref() == Some(&json!(stream.next_after)) => + { + Some(stream) + } + other => { + discard_stream(&session.connection, other)?; + None + } + }; + let (adapter, connection) = (&session.adapter, &session.connection); let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); let consumption_columns = CONSUMPTION_COLUMN_TYPES.map(|(name, _)| name); let event_columns: Vec = match request { @@ -140,10 +199,10 @@ fn read_page( let event_columns: Vec<&str> = event_columns.iter().map(String::as_str).collect(); let (location, after, since, limit, table, columns, timestamp) = match request { WarehouseHistoryRead::Statistics(read) => { - return read_statistics(py, (adapter, connection), read); + return Ok((read_statistics(py, (adapter, connection), read)?, None)); } WarehouseHistoryRead::Catalog(read) => { - return read_catalog(py, (adapter, connection), read); + return Ok((read_catalog(py, (adapter, connection), read)?, None)); } WarehouseHistoryRead::SourceFreshness(read) => ( &read.location, @@ -242,6 +301,9 @@ fn read_page( "metadata page limit is outside the supported bounds", )); } + if let Some(stream) = continued { + return stream_page(py, (session, request), stream, limit); + } let database = location.database.as_ref().map_or_else( || py.None().into_bound(py), |value| PyString::new(py, value).into_any(), @@ -259,12 +321,13 @@ fn read_page( )? .is_truthy()?; if !exists { - return Ok(WarehouseHistoryPage { + let page = WarehouseHistoryPage { records: Vec::new(), next_after: None, change_token: None, unchanged: false, - }); + }; + return Ok((page, None)); } let change_token: Option = if after.is_none() { relation_change_token(py, (adapter, connection), (&database, &schema, &name))? @@ -272,12 +335,13 @@ fn read_page( None }; if change_token.is_some() && change_token.as_deref() == unchanged_token(request) { - return Ok(WarehouseHistoryPage { + let page = WarehouseHistoryPage { records: Vec::new(), next_after: None, change_token, unchanged: true, - }); + }; + return Ok((page, None)); } let catalog_columns = call_with( &adapter.getattr("get_columns")?, @@ -408,6 +472,11 @@ fn read_page( }) .collect(); let projection: Vec<&str> = projection.iter().map(String::as_str).collect(); + let window = if session.local { + limit + } else { + limit * HISTORY_STREAM_PAGES + }; let sql = read_history_sql( &qualified, after.map(|_| literals.as_slice()), @@ -418,31 +487,61 @@ fn read_page( observed_since: observed_since.as_deref(), adapter: &adapter_name, has_event_id, - limit, + limit: window, }, ) .map_err(PyValueError::new_err)?; - let result = call_with( - &adapter.getattr("query")?, + let cursor = call_with( + &adapter.getattr("execute")?, &[ ("connection", connection.clone()), ("sql", PyString::new(py, &sql).into_any()), - ( - "limit", - pyo3::IntoPyObjectExt::into_bound_py_any(limit, py)?, - ), ], )?; + let stream = HistoryStream { + scan, + cursor, + next_after: Vec::new(), + remaining: window, + qualified, + columns: columns.iter().map(|column| (*column).to_owned()).collect(), + timestamp: timestamp.to_owned(), + cursor_indexes, + since, + }; + let (mut page, stream) = stream_page(py, (session, request), stream, limit)?; + page.change_token = change_token; + Ok((page, stream)) +} + +/// Fetch one page from the scan's result; a full page keeps the stream open for the next. +fn stream_page<'py>( + py: Python<'py>, + (session, request): (&HistorySession<'py>, &WarehouseHistoryRead), + mut stream: HistoryStream<'py>, + limit: usize, +) -> PyResult<(WarehouseHistoryPage, Option>)> { + let columns: Vec<&str> = stream.columns.iter().map(String::as_str).collect(); + let tuple = py.import("builtins")?.getattr("tuple")?; let mut records: Vec = Vec::new(); let mut next_after = None; let mut read_count = 0; - for row in field(&result, "rows")?.try_iter()? { + let fetch = limit.min(stream.remaining); + for row in stream + .cursor + .call_method1("fetchmany", (fetch,))? + .try_iter()? + { read_count += 1; - let row = row?; - let record = decode_record(py, (request, &row, &qualified), (columns, timestamp))?; - let mut cursor = vec![None; cursor_indexes.len()]; + let row = tuple.call1((row?,))?; + let record = decode_record( + py, + (request, &row, &stream.qualified), + (&columns, &stream.timestamp), + )?; + let mut cursor = vec![None; stream.cursor_indexes.len()]; for (index, value) in cursor.iter_mut().enumerate() { - let raw = row.get_item(cursor_indexes[index])?; + let raw = row.get_item(stream.cursor_indexes[index])?; *value = if raw.is_none() { None } else { @@ -466,20 +565,64 @@ fn read_page( WarehouseHistoryRecord::NodeResult(record) => record.ts, WarehouseHistoryRecord::Consumption(record) => record.recorded_at, }; - if since.is_none_or(|since| observed >= since) { + if stream.since.is_none_or(|since| observed >= since) { records.push(record); } next_after = Some(cursor); } - if read_count < limit { - next_after = None; - } - Ok(WarehouseHistoryPage { + stream.remaining -= read_count; + let open = match &next_after { + Some(after) if read_count == fetch && stream.remaining > 0 => { + stream.next_after.clone_from(after); + Some(stream) + } + Some(_) if read_count == fetch => { + discard_stream(&session.connection, Some(stream))?; + None + } + _ => { + next_after = None; + discard_stream(&session.connection, Some(stream))?; + None + } + }; + let page = WarehouseHistoryPage { records, next_after, - change_token, + change_token: None, unchanged: false, - }) + }; + Ok((page, open)) +} + +/// A scan's identity without its position, and the position, so a next page continues its result. +fn scan_identity(request: &WarehouseHistoryRead) -> PyResult<(Value, Option)> { + let mut value = + serde_json::to_value(request).map_err(|error| PyValueError::new_err(error.to_string()))?; + let mut after = None; + if let Some(read) = value + .as_object_mut() + .and_then(|kinds| kinds.values_mut().next()) + .and_then(Value::as_object_mut) + { + after = read.remove("after").filter(|after| !after.is_null()); + read.remove("limit"); + read.remove("unchanged_token"); + } + Ok((value, after)) +} + +/// Free the driver statement (SQL Server allows one active result); DuckDB's is the connection. +fn discard_stream( + connection: &Bound<'_, PyAny>, + stream: Option>, +) -> PyResult<()> { + if let Some(stream) = stream + && !stream.cursor.is(connection) + { + stream.cursor.call_method0("close")?; + } + Ok(()) } /// The token a complete earlier read recorded; only state-table scans carry one. diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 84ddc86e93..e59d7ec479 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -1295,24 +1295,36 @@ def test_given_change_tokens_when_a_served_location_resyncs_then_only_changed_or hook.mkdir() tokens: Path = hook / "tokens.json" reads: Path = hook / "reads.log" + sessions: Path = hook / "sessions.log" tokens.write_text( json.dumps({"_sqlbuild_fingerprints": "f1", "_sqlbuild_source_freshness": "s1"}) ) (hook / "sitecustomize.py").write_text( - "import json, re\n" + "import json, os, re\n" "from pathlib import Path\n" "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n" + "def log_session(event):\n" + f" with Path({str(sessions)!r}).open('a') as log:\n" + " log.write(f'{event} {os.getpid()}\\n')\n" + "original_connect, original_close = DuckDbAdapter.connect, DuckDbAdapter.close\n" + "def connect(self, config):\n" + " log_session('connect')\n" + " return original_connect(self, config)\n" + "def close(self, connection):\n" + " log_session('close')\n" + " return original_close(self, connection)\n" + "DuckDbAdapter.connect, DuckDbAdapter.close = connect, close\n" "def read_relation_change_token(self, *, connection, database, schema, name):\n" f" return json.loads(Path({str(tokens)!r}).read_text()).get(name)\n" "DuckDbAdapter.read_relation_change_token = read_relation_change_token\n" - "original_query = DuckDbAdapter.query\n" - "def query(self, *, connection, sql, limit):\n" + "original_execute = DuckDbAdapter.execute\n" + "def execute(self, *, connection, sql):\n" " read = re.search(r'FROM\\s+\\S*(_sqlbuild_\\w+)', sql)\n" " if read:\n" f" with Path({str(reads)!r}).open('a') as log:\n" " log.write(read.group(1) + '\\n')\n" - " return original_query(self, connection=connection, sql=sql, limit=limit)\n" - "DuckDbAdapter.query = query\n" + " return original_execute(self, connection=connection, sql=sql)\n" + "DuckDbAdapter.execute = execute\n" ) mirrors: tuple[str, str] = ("sqb_warehouse_fingerprints", "sqb_warehouse_freshness") with snapshot_api(tmp_path / "control") as api: @@ -1355,12 +1367,26 @@ def resync() -> list[str]: """Start one synchronization after a finished run and return the tables it read.""" completed: int = log.read_text().count("synchronization completed") reads.write_text("") + sessions.write_text("") record_finished_run(control) deadline: float = time.monotonic() + 45 while log.read_text().count("synchronization completed") == completed: assert location.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() time.sleep(0.2) + deadline = time.monotonic() + 10 + while sessions.read_text().count("close") == 0: + assert time.monotonic() < deadline, sessions.read_text() + time.sleep(0.1) + events: list[list[str]] = [ + line.split() for line in sessions.read_text().splitlines() + ] + assert len({pid for _, pid in events}) == 1, ( + "one warm interpreter reads every page of a synchronization" + ) + assert [event for event, _ in events] == ["connect", "close"] * ( + len(events) // 2 + ), "a local database file is never held open between pages, so writers proceed" return sorted(set(reads.read_text().split())) deadline: float = time.monotonic() + 45 @@ -1373,25 +1399,30 @@ def resync() -> list[str]: assert resync() == [], "unchanged tokens read nothing" assert mirrored() == initial + changed_rows: int = 250 connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) _ = adapter.execute( connection=connection, sql=( - "INSERT INTO _sqlbuild_fingerprints SELECT node_type,'orders-changed'," - "target_database,target_schema,target_name,'changed-run',definition_hash," - "version_hash,schema_fingerprint,definition_b64,metadata_json_b64,ts " - f"FROM _sqlbuild_fingerprints WHERE run_id='{test_case.expected_run_id}'" + "INSERT INTO _sqlbuild_fingerprints SELECT node_type," + "'orders-changed-' || page.i,target_database,target_schema,target_name," + "'changed-run',definition_hash,version_hash,schema_fingerprint," + "definition_b64,metadata_json_b64,ts FROM _sqlbuild_fingerprints " + f"CROSS JOIN range({changed_rows}) AS page(i) " + f"WHERE run_id='{test_case.expected_run_id}'" ), ) adapter.close(connection) tokens.write_text( json.dumps({"_sqlbuild_fingerprints": "f2", "_sqlbuild_source_freshness": "s1"}) ) - assert resync() == ["_sqlbuild_fingerprints"], "only the changed table is read" - assert mirrored() == (initial[0] + 1, initial[1]) + assert resync() == ["_sqlbuild_fingerprints"], ( + "only the changed table is read, across several pages" + ) + assert mirrored() == (initial[0] + changed_rows, initial[1]) with closing(sqlite3.connect(control)) as database: _ = database.execute(f"DELETE FROM {mirrors[1]}") database.commit() assert resync() == ["_sqlbuild_source_freshness"], "a lost mirror is read again" - assert mirrored() == (initial[0] + 1, initial[1]) + assert mirrored() == (initial[0] + changed_rows, initial[1])