diff --git a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json index ae0af98857..3528abd7e4 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json @@ -18,7 +18,7 @@ "held_write_lock": { "duckdb": [ { - "held": {"failure": "busy", "tables": ["source_freshness"]}, + "held": {"failure": "busy", "tables": ["fingerprints"]}, "released": {"failure": null, "tables": ["fingerprints", "source_freshness"]} } ], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index efec73492f..cad532ea86 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -54,7 +54,7 @@ use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::process::{Child, Command}; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinSet; -use tokio::time::{Instant, MissedTickBehavior}; +use tokio::time::{Instant, Interval, MissedTickBehavior}; use uuid::Uuid; use crate::argv::models::{Namespace, Stream, Value}; @@ -86,8 +86,8 @@ use crate::native_commands::main::_run_control::run_control; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, - LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, MetadataReply, - NativeCommand, NativeInvocation, ProjectLocation, ScanToken, + LocationMetadataPage, LocationMetadataSync, LocationServe, LocationWatch, LostStdout, + MetadataReply, NativeCommand, NativeInvocation, ProjectLocation, ScanToken, }; use crate::native_commands::types::LocationPublication; use crate::workers::main::docker_image_sources::docker_image_sources; @@ -1310,3 +1310,28 @@ async fn exchange_metadata_read( .await; (reader, result) } + +/// Wait for the next tick unless a synchronization's next page is due; `Some` means shutdown. +pub(crate) async fn next_turn( + clock: &mut Interval, + (published, watch, metadata): (Option, &LocationWatch, &Option), + stopped: impl Future>, +) -> Option> { + if published.is_some() + && watch.attempted.as_ref() == Some(&watch.observed) + && metadata.as_ref().is_some_and(LocationMetadataSync::paging) + { + return None; + } + tokio::select! { + result = stopped => Some(result), + _ = clock.tick() => None, + } +} + +/// The location's 250 ms turn; a slow turn skips missed ticks instead of bursting. +pub(crate) fn location_clock() -> Interval { + let mut clock = tokio::time::interval(Duration::from_millis(250)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + clock +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index e64efaba99..58cf25550f 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -12,6 +12,7 @@ use sqlbuild_core::git_revisions::main::head_commit::head_commit; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use tokio::time::Instant; +use crate::native_commands::constants::LOCATION_COMMIT_CHECK_SECONDS; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{LocationWatch, ProjectLocation}; use crate::replay::main::output_file_identities::output_file_identities; @@ -25,12 +26,16 @@ pub(crate) fn location_watch(root: &Path) -> LocationWatch { .collect(); let local_files = location_database_files(root, files); LocationWatch { - observed: location_inputs(root, &rules, &local_files), + observed: ( + location_files(root, &rules, &local_files), + head_commit(root), + ), rules, local_files, attempted: None, changed_at: tokio::time::Instant::now(), retry_at: None, + commit_checked_at: tokio::time::Instant::now(), } } @@ -39,7 +44,18 @@ pub(crate) fn observe_location_inputs( root: &Path, mut watch: LocationWatch, ) -> (LocationWatch, bool) { - let current = location_inputs(root, &watch.rules, &watch.local_files); + let commit = if watch.commit_checked_at.elapsed() + >= Duration::from_secs(LOCATION_COMMIT_CHECK_SECONDS) + { + watch.commit_checked_at = Instant::now(); + head_commit(root) + } else { + watch.observed.1.clone() + }; + let current = ( + location_files(root, &watch.rules, &watch.local_files), + commit, + ); if watch.observed != current { watch.observed = current; watch.changed_at = Instant::now(); @@ -74,12 +90,12 @@ fn location_rules() -> SnapshotRules { } } -/// Git commits matter to publication metadata even when no working-tree file changes. -pub(crate) fn location_inputs( +/// The project files a location publishes; the watch adds the git commit on its own cadence. +pub(crate) fn location_files( root: &Path, rules: &SnapshotRules, local_files: &[PathBuf], -) -> (Vec, Option) { +) -> Vec { let mut stamps = snapshot_project_files(root, rules); stamps.retain(|stamp| { !rules @@ -107,7 +123,7 @@ pub(crate) fn location_inputs( }); } stamps.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); - (stamps, head_commit(root)) + stamps } /// Apply resolved exclusions to retained observations, never a fresh snapshot of possible source edits. diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index a4ed2b6af9..6ca3f7a7b5 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -94,6 +94,8 @@ pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; pub(crate) const METADATA_POLL_SECONDS: u64 = 30; /// A metadata reader that ignores end of input this long after a synchronization is killed. pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5; +/// How often a location asks git whether `HEAD` moved. +pub(crate) const LOCATION_COMMIT_CHECK_SECONDS: u64 = 2; /// How often an idle location asks the server, never the warehouse, whether a run ended. pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5; /// Full key-set reconciliation is independent of the incremental poll and overlap window. diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index d5ee1a5c47..790d880b25 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -18,7 +18,7 @@ use crate::native_commands::_helpers::execution::{ metadata_health, next_scan, statistics_page, }; use crate::native_commands::_helpers::execution::{finish_metadata_page, read_metadata_page}; -use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::paths::location_files; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::constants::METADATA_POLL_SECONDS; use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; @@ -32,7 +32,8 @@ pub(super) async fn sync_location_metadata( ) -> Result, ControlError> { if Instant::now() < state.next_at || state.locations.is_empty() - || location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed + || location_files(&options.project_dir, &watch.rules, &watch.local_files) + != watch.observed.0 { return Ok(Some(state)); } @@ -80,7 +81,8 @@ pub(super) async fn sync_location_metadata( let Some(page) = page else { return failed_read(state, (client, poll), started, stopped.as_mut()).await; }; - if location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed + if location_files(&options.project_dir, &watch.rules, &watch.local_files) + != watch.observed.0 { return Ok(Some(state)); } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index cc51433faa..8b40bcfe0b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -15,7 +15,7 @@ use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::check_location_connection; -use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::paths::location_files; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::main::_location_sensor::evaluate_location_sensor; @@ -48,7 +48,8 @@ pub(super) async fn run_location_work( .unwrap_or_default(); let mut files: Vec = Vec::new(); let outcome = if generation == Some(claim.generation) - && location_inputs(&options.project_dir, &watch.rules, &watch.local_files) == watch.observed + && location_files(&options.project_dir, &watch.rules, &watch.local_files) + == watch.observed.0 { let mut shutting_down = false; let expires = async { @@ -73,7 +74,7 @@ pub(super) async fn run_location_work( .retain(|stamp| !files.contains(&options.project_dir.join(&stamp.relative_path))); let mut excluded = watch.local_files.clone(); excluded.extend(files.clone()); - if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected { + if location_files(&options.project_dir, &watch.rules, &excluded) == expected.0 { outcome } else { LocationWorkOutcome::Superseded {} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 30e7528589..c95603715d 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -10,13 +10,14 @@ use sqlbuild_orchestration::platform::models::{ }; use sqlbuild_orchestration::store::errors::ControlError; use tokio::sync::mpsc; -use tokio::time::{Instant, MissedTickBehavior}; +use tokio::time::Instant; use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - check_published_location, close_metadata_reader, compile_generation, location_generation, - location_heartbeats, metadata_sync, report_location_health, shutdown_signal, + check_published_location, close_metadata_reader, compile_generation, location_clock, + location_generation, location_heartbeats, metadata_sync, next_turn, report_location_health, + shutdown_signal, }; use crate::native_commands::_helpers::paths::{ exclude_location_databases, location_watch, observe_location_inputs, @@ -63,8 +64,7 @@ async fn watch_location( let mut work_at = Instant::now() + Duration::from_secs(2); let mut published_generation = None; let mut metadata = None; - let mut clock = tokio::time::interval(Duration::from_millis(250)); - clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut clock = location_clock(); let stopped = shutdown_signal(); tokio::pin!(stopped); drop(write_output(&[( @@ -72,9 +72,14 @@ async fn watch_location( "Starting code location; watching project inputs...\n", )])); loop { - tokio::select! { - result = &mut stopped => return result, - _ = clock.tick() => {}, + if let Some(result) = next_turn( + &mut clock, + (published_generation, &watch, &metadata), + stopped.as_mut(), + ) + .await + { + return result; } let ready; (watch, ready) = observe_location_inputs(&options.project_dir, watch); diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f242f0d0f6..9a762fcf63 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -163,6 +163,8 @@ pub(crate) struct LocationWatch { pub(crate) attempted: Option<(Vec, Option)>, pub(crate) changed_at: tokio::time::Instant, pub(crate) retry_at: Option, + /// Git commits matter to publication metadata, but asking git every tick spawns a process each time. + pub(crate) commit_checked_at: tokio::time::Instant, } /// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. @@ -292,6 +294,11 @@ impl LocationMetadataSync { + std::time::Duration::from_secs(super::constants::METADATA_POLL_SECONDS); } + /// A synchronization is mid-flight and its next page is due, so the loop need not wait a tick. + pub(crate) fn paging(&self) -> bool { + !self.idle && tokio::time::Instant::now() >= self.next_at + } + pub(crate) fn reconciling(&self) -> bool { self.kind() != WarehouseHistoryKind::Statistics && (self.reconcile || self.kind() == WarehouseHistoryKind::Catalog) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 884eaeaf23..b230cebb88 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -211,6 +211,20 @@ class StateEventsTestCase(CatalogConnectionTestCase): legacy_columns: tuple[str, ...] = () +@dataclass(frozen=True) +class SyncPacingTestCase: + """A tick-paced sync waits `added_rows / 100` ticks of 250 ms; git is asked every 2 s.""" + + description: str + adapter_name: str + connection_key: str + expected_run_id: str + added_rows: int + idle_seconds: float + expected_max_seconds: float + expected_max_git_calls: int + + @dataclass(frozen=True) class AutomaticHistoryTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 2e4d50dc5f..f27869d557 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -299,7 +299,7 @@ def append_consumption_history(adapter: Any, connection: Any, fixture: dict[str, columns: list[str] = list(fixture["record"])[2:] types: dict[str, StateSqlValueType] = dict.fromkeys(columns, StateSqlValueType.STRING) | { "input_count": StateSqlValueType.INTEGER, - "recorded_at": StateSqlValueType.TIMESTAMP, + "recorded_at": StateSqlValueType.TEXT_TIMESTAMP, } rows: list[dict[str, Any]] = [ row | {"recorded_at": datetime.fromisoformat(row["recorded_at"])} @@ -525,6 +525,18 @@ def wait_for_log( time.sleep(0.02) +def wait_for_idle_sync(process: subprocess.Popen[bytes], log: Path) -> None: + """Wait until the latest synchronization has completed, so a held lock meets the next one.""" + + deadline: float = time.monotonic() + 45 + text: str = log.read_text() + while text.rfind("synchronization completed.") < text.rfind("Reading warehouse history"): + assert process.poll() is None, text + assert time.monotonic() < deadline, text + time.sleep(0.05) + text = log.read_text() + + def location_health_reports(api: SnapshotApi) -> list[dict[str, Any]]: request: Request = Request( f"{api.url}/api/locations/{api.location_id}/health", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index c07d61f44e..94c53c35f6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -5,6 +5,7 @@ import hashlib import json import os +import shutil import sqlite3 import sys import time @@ -42,6 +43,7 @@ CatalogConnectionTestCase, FingerprintHistoryTestCase, StateEventsTestCase, + SyncPacingTestCase, WarehouseHistoryTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( @@ -67,6 +69,7 @@ snapshot_api, sync_warehouse_history, wait_for_generation, + wait_for_idle_sync, wait_for_location_work, wait_for_warehouse_sync_health, warehouse_history_page, @@ -350,6 +353,14 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and "external-run", (), ), + ( + "sqlite", + "path", + "consumption_history", + append_consumption_history, + "external-run", + (), + ), ( "duckdb", "database", @@ -1154,6 +1165,7 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre assert (project / "warehouse.db").exists() == availability["available"] held_locks: list[dict[str, Any]] = fixture["held_write_lock"][test_case.adapter_name] for held_lock in held_locks: + wait_for_idle_sync(location, log) lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") record_finished_run(control) reports = wait_for_warehouse_sync_health( @@ -1482,3 +1494,123 @@ def resync() -> list[str]: database.commit() assert resync() == ["_sqlbuild_source_freshness"], "a lost mirror is read again" assert mirrored() == (initial[0] + changed_rows, initial[1]) + + +@pytest.mark.parametrize( + "test_case", + [ + SyncPacingTestCase( + "SQLite pages back to back", + "sqlite", + "path", + "legacy-run", + added_rows=4000, + idle_seconds=4.0, + expected_max_seconds=9.5, + expected_max_git_calls=4, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_a_sync_has_many_pages_then_pages_run_back_to_back_and_idle_git_checks_are_rare( + test_case: SyncPacingTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json" + ).read_text() + ) + write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) + shim: Path = tmp_path / "git-shim" + shim.mkdir() + calls: Path = shim / "calls.log" + calls.write_text("") + (shim / "git").write_text(f'#!/bin/sh\necho "$*" >> {calls}\nexec {shutil.which("git")} "$@"\n') + (shim / "git").chmod(0o755) + with snapshot_api(tmp_path / "control") as api: + log: Path = tmp_path / "location.log" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ + | { + "SQLBUILD_LOCATION_TOKEN": api.bearer, + "SQLBUILD_METADATA_RECONCILE_SECONDS": "3600", + "PATH": f"{shim}{os.pathsep}{os.environ['PATH']}", + }, + ) as location: + _ = wait_for_generation(api, "development", 1, location, log) + + def synced(completed: int) -> None: + deadline: float = time.monotonic() + 60 + while log.read_text().count("synchronization completed") == completed: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.01) + + synced(0) + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute( + connection=connection, + sql=( + "INSERT INTO _sqlbuild_fingerprints SELECT node_type," + "'orders-paced-' || page.i,target_database,target_schema,target_name," + "'paced-run',definition_hash,version_hash,schema_fingerprint," + "definition_b64,metadata_json_b64,ts FROM _sqlbuild_fingerprints " + "CROSS JOIN (WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i + 1 FROM n " + f"WHERE i < {test_case.added_rows - 1}) SELECT i FROM n) AS page " + f"WHERE run_id='{test_case.expected_run_id}'" + ), + ) + adapter.close(connection) + completed: int = log.read_text().count("synchronization completed") + reading: int = log.read_text().count("Reading warehouse history") + record_finished_run(control) + deadline: float = time.monotonic() + 30 + while log.read_text().count("Reading warehouse history") == reading: + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.01) + started: float = time.monotonic() + synced(completed) + elapsed: float = time.monotonic() - started + with closing(sqlite3.connect(control)) as database: + paced: int = database.execute( + "SELECT COUNT(*) FROM sqb_warehouse_fingerprints " + "WHERE json_extract(record_json, '$.run_id') = 'paced-run'" + ).fetchone()[0] + assert paced == test_case.added_rows, test_case.description + assert elapsed < test_case.expected_max_seconds, ( + f"{test_case.description}: {elapsed:.1f} s for " + f"{test_case.added_rows // 100} pages; a tick per page needs at least " + f"{test_case.added_rows / 400:.0f} s" + ) + + calls.write_text("") + time.sleep(test_case.idle_seconds) + idle_calls: list[str] = calls.read_text().splitlines() + assert len(idle_calls) <= test_case.expected_max_git_calls, ( + f"{test_case.description}: an idle location ran git {len(idle_calls)} times " + f"in {test_case.idle_seconds} s" + )