Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 153 additions & 3 deletions crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ use sqlbuild_orchestration::webhooks::main::{
connect_webhook::connect_webhook, run_webhook::run_webhook,
};
use sqlbuild_orchestration::webhooks::models::{Webhook, WebhookSource};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::process::{Child, Command};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinSet;
Expand All @@ -72,8 +72,9 @@ use crate::native_commands::_helpers::terminal::{
};
use crate::native_commands::constants::{
ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_OVERLAP_SECONDS,
METADATA_POLL_SECONDS, METADATA_RECONCILE_SECONDS, METADATA_SYNC_CHECK_SECONDS,
SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND,
METADATA_POLL_SECONDS, METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS,
METADATA_REPLY_BYTES, METADATA_SYNC_CHECK_SECONDS, SCHEDULER_COMMAND,
UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND,
};
use crate::native_commands::errors::PythonNeeded;
use crate::native_commands::main::_run_admin::run_admin;
Expand Down Expand Up @@ -358,6 +359,7 @@ pub(crate) fn metadata_sync(
scan_token: None,
scan_delivered: false,
mirror_holds_rows: false,
reader: None,
}
}

Expand Down Expand Up @@ -1109,3 +1111,151 @@ pub(crate) fn run_command(
pub(crate) fn plan_failure(message: &str) -> serde_json::Value {
json!({"error": {"code": "plan_failed", "message": message, "help": null}})
}

pub(crate) fn open_location_runtime(
options: &LocationServe,
script: &Path,
python_sources: Vec<sqlbuild_cache::project_snapshot::models::PathStamp>,
) -> Result<LocationCompiler, ControlError> {
let mut child = location_runtime_child(options, script)?;
let stdin = child
.stdin
.take()
.ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?;
let stdout = child.stdout.take().ok_or_else(|| {
ControlError::Invalid("location interpreter output is unavailable".into())
})?;
Ok(LocationCompiler {
child,
stdin,
stdout: BufReader::new(stdout),
python_sources,
})
}

pub(crate) async fn stop_location_runtime(
runtime: Option<LocationCompiler>,
) -> Result<(), ControlError> {
if let Some(mut runtime) = runtime {
if runtime.child.try_wait()?.is_none() {
runtime.child.kill().await?;
} else {
runtime.child.wait().await?;
}
}
Ok(())
}

/// An outer absence means shutdown; an inner absence is a failed read, never an empty table.
pub(crate) async fn read_metadata_page(
(options, script): (&LocationServe, &Path),
mut state: LocationMetadataSync,
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<(LocationMetadataSync, Option<WarehouseHistoryPage>)>, ControlError> {
if let Some(statistics) = &state.statistics
&& statistics.names.is_empty()
{
let page = WarehouseHistoryPage {
records: Vec::new(),
next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]),
change_token: None,
unchanged: false,
};
return Ok(Some((state, Some(page))));
}
let request = history_request(&state)?;
let reader = match state.reader.take() {
Some(reader) => reader,
None => match open_location_runtime(options, script, Vec::new()) {
Ok(reader) => reader,
Err(_) => return Ok(Some((state, None))),
},
};
tokio::pin!(stopped);
tokio::select! {
result = &mut stopped => {
result?;
Ok(None)
},
result = tokio::time::timeout(
Duration::from_secs(METADATA_POLL_SECONDS),
exchange_metadata_read(reader, request),
) => match result {
Ok((reader, Ok(page))) => {
state.reader = Some(reader);
Ok(Some((state, page)))
}
Ok((reader, Err(_))) => {
stop_location_runtime(Some(reader)).await?;
Ok(Some((state, None)))
}
Err(_) => Ok(Some((state, None))),
},
}
}

/// A completed synchronization closes its reader, so no warehouse session outlives the sync.
pub(crate) async fn finish_metadata_page(
state: LocationMetadataSync,
) -> Result<Option<LocationMetadataSync>, ControlError> {
if state.idle {
return Ok(Some(close_metadata_reader(state).await?));
}
Ok(Some(state))
}

/// End of input lets the reader close its warehouse session; one that ignores it is killed.
pub(crate) async fn close_metadata_reader(
mut state: LocationMetadataSync,
) -> Result<LocationMetadataSync, ControlError> {
if let Some(LocationCompiler {
mut child, stdin, ..
}) = state.reader.take()
{
drop(stdin);
match tokio::time::timeout(
Duration::from_secs(METADATA_READER_CLOSE_SECONDS),
child.wait(),
)
.await
{
Ok(waited) => drop(waited?),
Err(_) => child.kill().await?,
}
}
Ok(state)
}

/// A reply of `null` means the reader's read failed and it dropped its warehouse session.
async fn exchange_metadata_read(
mut reader: LocationCompiler,
request: Vec<u8>,
) -> (
LocationCompiler,
Result<Option<WarehouseHistoryPage>, ControlError>,
) {
let result = async {
reader.stdin.write_all(&request).await?;
reader.stdin.write_all(b"\n").await?;
reader.stdin.flush().await?;
let mut line = String::new();
let read = (&mut reader.stdout)
.take(METADATA_REPLY_BYTES + 1)
.read_line(&mut line)
.await?;
if read == 0 {
return Err(ControlError::Invalid(
"location metadata reader exited before replying".into(),
));
}
if read as u64 > METADATA_REPLY_BYTES || !line.ends_with('\n') {
return Err(ControlError::Invalid(
"location reply exceeds its byte limit".into(),
));
}
serde_json::from_str(&line)
.map_err(|_| ControlError::Invalid("invalid location reply".into()))
}
.await;
(reader, result)
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30;
pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576;
/// Read timeout, and the delay before retrying a failed page of an unfinished synchronization.
pub(crate) const METADATA_POLL_SECONDS: u64 = 30;
/// A metadata reader that ignores end of input this long after a synchronization is killed.
pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5;
/// How often an idle location asks the server, never the warehouse, whether a run ended.
pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5;
/// Full key-set reconciliation is independent of the incremental poll and overlap window.
Expand Down
50 changes: 10 additions & 40 deletions crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,10 @@
use std::path::Path;

use sqlbuild_orchestration::store::errors::ControlError;
use tokio::io::BufReader;
use tokio::sync::mpsc;

use crate::native_commands::_helpers::execution::{
exchange_location_generation, location_runtime_child,
};
use crate::native_commands::_helpers::execution::exchange_location_generation;
use crate::native_commands::_helpers::execution::{open_location_runtime, stop_location_runtime};
use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe};

pub(crate) async fn run_location_compiler(
Expand All @@ -23,10 +21,14 @@ pub(crate) async fn run_location_compiler(
.as_ref()
.is_some_and(|compiler| compiler.python_sources != request.python_sources)
{
stop_compiler(compiler.take()).await?;
stop_location_runtime(compiler.take()).await?;
}
if compiler.is_none() {
compiler = Some(open_compiler(options, script, request.python_sources)?);
compiler = Some(open_location_runtime(
options,
script,
request.python_sources,
)?);
}
let active = compiler
.as_mut()
Expand All @@ -40,44 +42,12 @@ pub(crate) async fn run_location_compiler(
.reply
.send(result.map_err(|error| error.to_string()));
if failed {
stop_compiler(compiler.take()).await?;
stop_location_runtime(compiler.take()).await?;
}
}
Ok(())
}
.await;
stop_compiler(compiler).await?;
stop_location_runtime(compiler).await?;
result
}

fn open_compiler(
options: &LocationServe,
script: &Path,
python_sources: Vec<sqlbuild_cache::project_snapshot::models::PathStamp>,
) -> Result<LocationCompiler, ControlError> {
let mut child = location_runtime_child(options, script)?;
let stdin = child
.stdin
.take()
.ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?;
let stdout = child.stdout.take().ok_or_else(|| {
ControlError::Invalid("location interpreter output is unavailable".into())
})?;
Ok(LocationCompiler {
child,
stdin,
stdout: BufReader::new(stdout),
python_sources,
})
}

async fn stop_compiler(compiler: Option<LocationCompiler>) -> Result<(), ControlError> {
if let Some(mut compiler) = compiler {
if compiler.child.try_wait()?.is_none() {
compiler.child.kill().await?;
} else {
compiler.child.wait().await?;
}
}
Ok(())
}
64 changes: 11 additions & 53 deletions crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,18 @@ use sqlbuild_orchestration::platform::models::{
LocationWorkClient, LocationWorkPoll, WarehouseSyncFailure,
};
use sqlbuild_orchestration::store::errors::ControlError;
use sqlbuild_orchestration::warehouse_metadata::models::{
WarehouseHistoryKind, WarehouseHistoryPage,
};
use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryKind;
use tokio::time::Instant;

use crate::argv::models::Stream;
use crate::native_commands::_helpers::execution::{
acknowledge_metadata_page, begin_after_run, freeze_history_page, history_request,
load_watermark, location_result, metadata_health, next_scan, statistics_page,
acknowledge_metadata_page, begin_after_run, freeze_history_page, load_watermark,
metadata_health, next_scan, statistics_page,
};
use crate::native_commands::_helpers::execution::{finish_metadata_page, read_metadata_page};
use crate::native_commands::_helpers::paths::location_inputs;
use crate::native_commands::_helpers::terminal::write_output;
use crate::native_commands::constants::{METADATA_POLL_SECONDS, METADATA_REPLY_BYTES};
use crate::native_commands::constants::METADATA_POLL_SECONDS;
use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch};

/// One bounded source or mirror page leaves named work polling a turn between pages.
Expand Down Expand Up @@ -72,10 +71,12 @@ pub(super) async fn sync_location_metadata(
return retry_metadata(state, error);
}
}
let Some(page) = read_metadata_page((options, script), &state, stopped.as_mut()).await?
let Some((read, page)) =
read_metadata_page((options, script), state, stopped.as_mut()).await?
else {
return Ok(None);
};
state = read;
let Some(page) = page else {
tokio::select! {
result = &mut stopped => { result?; return Ok(None); },
Expand All @@ -97,7 +98,7 @@ pub(super) async fn sync_location_metadata(
result = &mut stopped => { result?; return Ok(None); },
() = metadata_health(&state, (client, poll), (started, None)) => {},
}
return Ok(Some(next_scan(state)));
return finish_metadata_page(next_scan(state)).await;
}
state = freeze_history_page(state, page, poll)?;
}
Expand All @@ -119,7 +120,7 @@ pub(super) async fn sync_location_metadata(
result = &mut stopped => { result?; return Ok(None); },
() = metadata_health(&state, (client, poll), (started, None)) => {},
}
Ok(Some(acknowledge_metadata_page(state)))
finish_metadata_page(acknowledge_metadata_page(state)).await
}

fn retry_metadata(
Expand All @@ -132,6 +133,7 @@ fn retry_metadata(
return Err(ControlError::Invalid(error.to_string()));
}
state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS);
state.reader = None;
drop(write_output(&[(
Stream::Stderr,
&format!(
Expand All @@ -140,47 +142,3 @@ fn retry_metadata(
)]));
Ok(Some(state))
}

/// An outer absence means shutdown; an inner absence is a failed read, never an empty table.
async fn read_metadata_page(
(options, script): (&LocationServe, &Path),
state: &LocationMetadataSync,
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<Option<WarehouseHistoryPage>>, ControlError> {
if let Some(statistics) = &state.statistics
&& statistics.names.is_empty()
{
return Ok(Some(Some(WarehouseHistoryPage {
records: Vec::new(),
next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]),
change_token: None,
unchanged: false,
})));
}
let (mut child, reply) = match location_result::<Option<WarehouseHistoryPage>>(
(options, script),
(history_request(state)?, METADATA_REPLY_BYTES),
) {
Ok(exchange) => exchange,
Err(_) => return Ok(Some(None)),
};
tokio::pin!(stopped);
let outcome = tokio::select! {
result = &mut stopped => { child.kill().await?; result?; return Ok(None); },
result = tokio::time::timeout(Duration::from_secs(METADATA_POLL_SECONDS), async {
let page = reply.await?;
if !child.wait().await?.success() {
return Err(ControlError::Invalid("location metadata interpreter failed".into()));
}
Ok(page)
}) => result,
};
if child.try_wait()?.is_none() {
child.kill().await?;
}
let page = match outcome {
Ok(Ok(page)) => page,
Ok(Err(_)) | Err(_) => None,
};
Ok(Some(page))
}
Loading
Loading