Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"held_write_lock": {
"duckdb": [
{
"held": {"failure": "busy", "tables": ["source_freshness"]},
"held": {"failure": "busy", "tables": ["fingerprints"]},
"released": {"failure": null, "tables": ["fingerprints", "source_freshness"]}
}
],
Expand Down
31 changes: 28 additions & 3 deletions crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::process::{Child, Command};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinSet;
use tokio::time::{Instant, MissedTickBehavior};
use tokio::time::{Instant, Interval, MissedTickBehavior};
use uuid::Uuid;

use crate::argv::models::{Namespace, Stream, Value};
Expand Down Expand Up @@ -86,8 +86,8 @@ use crate::native_commands::main::_run_control::run_control;
use crate::native_commands::main::_run_cost::run_cost;
use crate::native_commands::models::{
CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler,
LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, MetadataReply,
NativeCommand, NativeInvocation, ProjectLocation, ScanToken,
LocationMetadataPage, LocationMetadataSync, LocationServe, LocationWatch, LostStdout,
MetadataReply, NativeCommand, NativeInvocation, ProjectLocation, ScanToken,
};
use crate::native_commands::types::LocationPublication;
use crate::workers::main::docker_image_sources::docker_image_sources;
Expand Down Expand Up @@ -1310,3 +1310,28 @@ async fn exchange_metadata_read(
.await;
(reader, result)
}

/// Wait for the next tick unless a synchronization's next page is due; `Some` means shutdown.
pub(crate) async fn next_turn(
clock: &mut Interval,
(published, watch, metadata): (Option<i64>, &LocationWatch, &Option<LocationMetadataSync>),
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Option<Result<(), ControlError>> {
if published.is_some()
&& watch.attempted.as_ref() == Some(&watch.observed)
&& metadata.as_ref().is_some_and(LocationMetadataSync::paging)
{
return None;
}
tokio::select! {
result = stopped => Some(result),
_ = clock.tick() => None,
}
}

/// The location's 250 ms turn; a slow turn skips missed ticks instead of bursting.
pub(crate) fn location_clock() -> Interval {
let mut clock = tokio::time::interval(Duration::from_millis(250));
clock.set_missed_tick_behavior(MissedTickBehavior::Skip);
clock
}
28 changes: 22 additions & 6 deletions crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use sqlbuild_core::git_revisions::main::head_commit::head_commit;
use sqlbuild_rules::project_tree::main::project_real_path::project_real_path;
use tokio::time::Instant;

use crate::native_commands::constants::LOCATION_COMMIT_CHECK_SECONDS;
use crate::native_commands::errors::PythonNeeded;
use crate::native_commands::models::{LocationWatch, ProjectLocation};
use crate::replay::main::output_file_identities::output_file_identities;
Expand All @@ -25,12 +26,16 @@ pub(crate) fn location_watch(root: &Path) -> LocationWatch {
.collect();
let local_files = location_database_files(root, files);
LocationWatch {
observed: location_inputs(root, &rules, &local_files),
observed: (
location_files(root, &rules, &local_files),
head_commit(root),
),
rules,
local_files,
attempted: None,
changed_at: tokio::time::Instant::now(),
retry_at: None,
commit_checked_at: tokio::time::Instant::now(),
}
}

Expand All @@ -39,7 +44,18 @@ pub(crate) fn observe_location_inputs(
root: &Path,
mut watch: LocationWatch,
) -> (LocationWatch, bool) {
let current = location_inputs(root, &watch.rules, &watch.local_files);
let commit = if watch.commit_checked_at.elapsed()
>= Duration::from_secs(LOCATION_COMMIT_CHECK_SECONDS)
{
watch.commit_checked_at = Instant::now();
head_commit(root)
} else {
watch.observed.1.clone()
};
let current = (
location_files(root, &watch.rules, &watch.local_files),
commit,
);
if watch.observed != current {
watch.observed = current;
watch.changed_at = Instant::now();
Expand Down Expand Up @@ -74,12 +90,12 @@ fn location_rules() -> SnapshotRules {
}
}

/// Git commits matter to publication metadata even when no working-tree file changes.
pub(crate) fn location_inputs(
/// The project files a location publishes; the watch adds the git commit on its own cadence.
pub(crate) fn location_files(
root: &Path,
rules: &SnapshotRules,
local_files: &[PathBuf],
) -> (Vec<PathStamp>, Option<String>) {
) -> Vec<PathStamp> {
let mut stamps = snapshot_project_files(root, rules);
stamps.retain(|stamp| {
!rules
Expand Down Expand Up @@ -107,7 +123,7 @@ pub(crate) fn location_inputs(
});
}
stamps.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
(stamps, head_commit(root))
stamps
}

/// Apply resolved exclusions to retained observations, never a fresh snapshot of possible source edits.
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576;
pub(crate) const METADATA_POLL_SECONDS: u64 = 30;
/// A metadata reader that ignores end of input this long after a synchronization is killed.
pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5;
/// How often a location asks git whether `HEAD` moved.
pub(crate) const LOCATION_COMMIT_CHECK_SECONDS: u64 = 2;
/// How often an idle location asks the server, never the warehouse, whether a run ended.
pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5;
/// Full key-set reconciliation is independent of the incremental poll and overlap window.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use crate::native_commands::_helpers::execution::{
metadata_health, next_scan, statistics_page,
};
use crate::native_commands::_helpers::execution::{finish_metadata_page, read_metadata_page};
use crate::native_commands::_helpers::paths::location_inputs;
use crate::native_commands::_helpers::paths::location_files;
use crate::native_commands::_helpers::terminal::write_output;
use crate::native_commands::constants::METADATA_POLL_SECONDS;
use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch};
Expand All @@ -32,7 +32,8 @@ pub(super) async fn sync_location_metadata(
) -> Result<Option<LocationMetadataSync>, ControlError> {
if Instant::now() < state.next_at
|| state.locations.is_empty()
|| location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed
|| location_files(&options.project_dir, &watch.rules, &watch.local_files)
!= watch.observed.0
{
return Ok(Some(state));
}
Expand Down Expand Up @@ -80,7 +81,8 @@ pub(super) async fn sync_location_metadata(
let Some(page) = page else {
return failed_read(state, (client, poll), started, stopped.as_mut()).await;
};
if location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed
if location_files(&options.project_dir, &watch.rules, &watch.local_files)
!= watch.observed.0
{
return Ok(Some(state));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use uuid::Uuid;

use crate::argv::models::Stream;
use crate::native_commands::_helpers::execution::check_location_connection;
use crate::native_commands::_helpers::paths::location_inputs;
use crate::native_commands::_helpers::paths::location_files;
use crate::native_commands::_helpers::terminal::write_output;
use crate::native_commands::main::_location_plan::plan_location;
use crate::native_commands::main::_location_sensor::evaluate_location_sensor;
Expand Down Expand Up @@ -48,7 +48,8 @@ pub(super) async fn run_location_work(
.unwrap_or_default();
let mut files: Vec<PathBuf> = Vec::new();
let outcome = if generation == Some(claim.generation)
&& location_inputs(&options.project_dir, &watch.rules, &watch.local_files) == watch.observed
&& location_files(&options.project_dir, &watch.rules, &watch.local_files)
== watch.observed.0
{
let mut shutting_down = false;
let expires = async {
Expand All @@ -73,7 +74,7 @@ pub(super) async fn run_location_work(
.retain(|stamp| !files.contains(&options.project_dir.join(&stamp.relative_path)));
let mut excluded = watch.local_files.clone();
excluded.extend(files.clone());
if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected {
if location_files(&options.project_dir, &watch.rules, &excluded) == expected.0 {
outcome
} else {
LocationWorkOutcome::Superseded {}
Expand Down
21 changes: 13 additions & 8 deletions crates/sqlbuild-cli/src/native_commands/main/_run_location.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ use sqlbuild_orchestration::platform::models::{
};
use sqlbuild_orchestration::store::errors::ControlError;
use tokio::sync::mpsc;
use tokio::time::{Instant, MissedTickBehavior};
use tokio::time::Instant;
use uuid::Uuid;

use crate::argv::models::Stream;
use crate::native_commands::_helpers::execution::{
check_published_location, close_metadata_reader, compile_generation, location_generation,
location_heartbeats, metadata_sync, report_location_health, shutdown_signal,
check_published_location, close_metadata_reader, compile_generation, location_clock,
location_generation, location_heartbeats, metadata_sync, next_turn, report_location_health,
shutdown_signal,
};
use crate::native_commands::_helpers::paths::{
exclude_location_databases, location_watch, observe_location_inputs,
Expand Down Expand Up @@ -63,18 +64,22 @@ async fn watch_location(
let mut work_at = Instant::now() + Duration::from_secs(2);
let mut published_generation = None;
let mut metadata = None;
let mut clock = tokio::time::interval(Duration::from_millis(250));
clock.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut clock = location_clock();
let stopped = shutdown_signal();
tokio::pin!(stopped);
drop(write_output(&[(
Stream::Stderr,
"Starting code location; watching project inputs...\n",
)]));
loop {
tokio::select! {
result = &mut stopped => return result,
_ = clock.tick() => {},
if let Some(result) = next_turn(
&mut clock,
(published_generation, &watch, &metadata),
stopped.as_mut(),
)
.await
{
return result;
}
let ready;
(watch, ready) = observe_location_inputs(&options.project_dir, watch);
Expand Down
7 changes: 7 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ pub(crate) struct LocationWatch {
pub(crate) attempted: Option<(Vec<PathStamp>, Option<String>)>,
pub(crate) changed_at: tokio::time::Instant,
pub(crate) retry_at: Option<tokio::time::Instant>,
/// Git commits matter to publication metadata, but asking git every tick spawns a process each time.
pub(crate) commit_checked_at: tokio::time::Instant,
}

/// A bounded supervisor request; dropping the reply cancels and reaps an active compiler.
Expand Down Expand Up @@ -292,6 +294,11 @@ impl LocationMetadataSync {
+ std::time::Duration::from_secs(super::constants::METADATA_POLL_SECONDS);
}

/// A synchronization is mid-flight and its next page is due, so the loop need not wait a tick.
pub(crate) fn paging(&self) -> bool {
!self.idle && tokio::time::Instant::now() >= self.next_at
}

pub(crate) fn reconciling(&self) -> bool {
self.kind() != WarehouseHistoryKind::Statistics
&& (self.reconcile || self.kind() == WarehouseHistoryKind::Catalog)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,20 @@ class StateEventsTestCase(CatalogConnectionTestCase):
legacy_columns: tuple[str, ...] = ()


@dataclass(frozen=True)
class SyncPacingTestCase:
"""A tick-paced sync waits `added_rows / 100` ticks of 250 ms; git is asked every 2 s."""

description: str
adapter_name: str
connection_key: str
expected_run_id: str
added_rows: int
idle_seconds: float
expected_max_seconds: float
expected_max_git_calls: int


@dataclass(frozen=True)
class AutomaticHistoryTestCase:
description: str
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ def append_consumption_history(adapter: Any, connection: Any, fixture: dict[str,
columns: list[str] = list(fixture["record"])[2:]
types: dict[str, StateSqlValueType] = dict.fromkeys(columns, StateSqlValueType.STRING) | {
"input_count": StateSqlValueType.INTEGER,
"recorded_at": StateSqlValueType.TIMESTAMP,
"recorded_at": StateSqlValueType.TEXT_TIMESTAMP,
}
rows: list[dict[str, Any]] = [
row | {"recorded_at": datetime.fromisoformat(row["recorded_at"])}
Expand Down Expand Up @@ -525,6 +525,18 @@ def wait_for_log(
time.sleep(0.02)


def wait_for_idle_sync(process: subprocess.Popen[bytes], log: Path) -> None:
"""Wait until the latest synchronization has completed, so a held lock meets the next one."""

deadline: float = time.monotonic() + 45
text: str = log.read_text()
while text.rfind("synchronization completed.") < text.rfind("Reading warehouse history"):
assert process.poll() is None, text
assert time.monotonic() < deadline, text
time.sleep(0.05)
text = log.read_text()


def location_health_reports(api: SnapshotApi) -> list[dict[str, Any]]:
request: Request = Request(
f"{api.url}/api/locations/{api.location_id}/health",
Expand Down
Loading
Loading