From 6a5d974df301c10cb91b431ce7b9c684cf37aa42 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:08:50 +0000 Subject: [PATCH] fix: stop a single custom-rule host at its next invocation when interrupted Ctrl-C during custom rules that run on one host, the default for small projects, waited until that host had run every invocation. Only split hosts had a cancellation marker; a single-host run, the single-host fallback after a failed split and the stateful rerun had none, and the fallback did not check for interrupts at all. Every host run now gets its own cancellation marker and waits for its host the way split hosts do, so an interrupt stops the host after the invocation in flight. Module-state detection, which the marker used to switch on, is now its own flag, still set only for split hosts. --- .../_helpers/rule_records/constants.rs | 7 +- .../src/rules/_helpers/custom/custom_hosts.rs | 100 +++++++++++------- src/sqlbuild/_native.pyi | 4 +- .../_helpers/host/custom_evaluation.py | 2 +- .../rule_engine/_helpers/host/custom_host.py | 5 +- .../rule_engine/_helpers/host/_test_types.py | 14 +++ .../_helpers/host/test_custom_host_pool.py | 71 +++++++++++++ 7 files changed, 158 insertions(+), 45 deletions(-) diff --git a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs index 989133a1cc..34f0f5b1bc 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs @@ -367,8 +367,11 @@ const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec { }; const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec { name: "CustomHostPartition", - doc: "How one host of a split run detects module state and learns the run was cancelled.", - fields: &[FieldSpec::required("cancelled")], + doc: "How a host learns of a cancelled run and whether it detects module state.", + fields: &[ + FieldSpec::required("cancelled"), + FieldSpec::new("detect_state", FieldDefault::False), + ], }; /// The rule engine value records. diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs index 532b3414d1..5849cad6a8 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs @@ -164,13 +164,12 @@ impl Launch { host_outcome(spec) } - /// The `run_custom_host_json` request for one host's plan and optional cancellation marker. - fn spec(&self, plan: &HostPlan, cancel_marker: Option<&Path>) -> String { + /// The host request for one plan and cancel marker; only split hosts detect module state. + fn spec(&self, plan: &HostPlan, cancel_marker: &Path, detect_state: bool) -> String { let mut payload: Map = self.payload.clone(); payload.insert("plan".to_owned(), json!(plan)); - if let Some(marker) = cancel_marker { - payload.insert("cancel_marker".to_owned(), json!(text(marker))); - } + payload.insert("cancel_marker".to_owned(), json!(text(cancel_marker))); + payload.insert("detect_state".to_owned(), json!(detect_state)); json!({ "program": self.program, "arguments": self.arguments, @@ -189,36 +188,23 @@ fn run_partitions<'py>( (input_dir, cancel): (&Path, &HostCancel), (write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path), ) -> PyResult> { - if plans.len() == 1 { - let spec: String = launch.spec(plan, None); - let outcomes: Vec = run_hosts( - py, - (launch, vec![spec]), - (write_inputs, abandoned), - (None, cancel), - )?; + let split: bool = plans.len() > 1; + let outcomes: Vec = run_cancellable( + py, + (launch, plans, split), + (input_dir, cancel), + (write_inputs, abandoned), + )?; + if !split { return outcomes .into_iter() .map(|outcome| outcome.map_err(|message| rules_error(py, message))) .collect(); } - let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?; - let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER); - let specs: Vec = plans - .iter() - .map(|partition| launch.spec(partition, Some(&marker))) - .collect(); - let outcomes: PyResult> = run_hosts( - py, - (launch, specs), - (write_inputs, abandoned), - (Some(&marker), cancel), - ); - let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir); - let outcomes: Vec = outcomes?; interrupted(py, cancel)?; if outcomes.iter().any(Result::is_err) { - return single_host(py, launch, plan).map(|slice| vec![slice]); + return single_host(py, (launch, plan), (input_dir, cancel), abandoned) + .map(|slice| vec![slice]); } let slices: Vec = outcomes.into_iter().filter_map(Result::ok).collect(); let stateful: BTreeSet = slices @@ -239,7 +225,7 @@ fn run_partitions<'py>( (code.clone(), subjects) }) .collect(); - let rerun: HostSlice = single_host(py, launch, &rerun_plan)?; + let rerun: HostSlice = single_host(py, (launch, &rerun_plan), (input_dir, cancel), abandoned)?; let mut merged: Vec = slices .into_iter() .map(|slice| without_codes(slice, &stateful)) @@ -248,19 +234,55 @@ fn run_partitions<'py>( Ok(merged) } -/// Run one host for `plan` once the payload already exists. -fn single_host(py: Python<'_>, launch: &Launch, plan: &HostPlan) -> PyResult { - let spec: String = launch.spec(plan, None); - py.detach(|| launch.run(&spec)) +/// Run one host for `plan` from its written payload; an interrupt stops its next invocation. +fn single_host( + py: Python<'_>, + (launch, plan): (&Launch, &HostPlan), + (input_dir, cancel): (&Path, &HostCancel), + abandoned: &Path, +) -> PyResult { + let outcomes: Vec = run_cancellable( + py, + (launch, std::slice::from_ref(plan), false), + (input_dir, cancel), + (|| Ok(()), abandoned), + )?; + outcomes + .into_iter() + .next() + .unwrap_or_else(|| Err("custom rule host returned no outcome".to_owned())) .map_err(|message| rules_error(py, message)) } +/// Run one host per plan with a fresh cancel marker that an interrupt or failed host sets. +fn run_cancellable( + py: Python<'_>, + (launch, plans, detect_state): (&Launch, &[HostPlan], bool), + (input_dir, cancel): (&Path, &HostCancel), + (write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path), +) -> PyResult> { + let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?; + let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER); + let specs: Vec = plans + .iter() + .map(|partition| launch.spec(partition, &marker, detect_state)) + .collect(); + let outcomes: PyResult> = run_hosts( + py, + (launch, specs), + (write_inputs, abandoned), + (&marker, cancel), + ); + let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir); + outcomes +} + /// Run every host on its own thread while this thread publishes the payload they wait for. fn run_hosts( py: Python<'_>, (launch, specs): (&Launch, Vec), (write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path), - (cancel_marker, cancel): (Option<&Path>, &HostCancel), + (cancel_marker, cancel): (&Path, &HostCancel), ) -> PyResult> { std::thread::scope(|scope| { let (sender, finished): (Sender, Receiver) = channel(); @@ -305,7 +327,7 @@ fn run_hosts( fn await_hosts( py: Python<'_>, (finished, count): (Receiver, usize), - cancel_marker: Option<&Path>, + cancel_marker: &Path, cancel: &HostCancel, ) -> PyResult<()> { let finished: Mutex> = Mutex::new(finished); @@ -341,11 +363,9 @@ fn interrupted(py: Python<'_>, cancel: &HostCancel) -> PyResult<()> { Ok(()) } -/// Stop split hosts at their next invocation; a running invocation still ends or times out. -fn cancel_hosts(cancel_marker: Option<&Path>) { - if let Some(marker) = cancel_marker { - let _touched: std::io::Result<()> = touch(marker); - } +/// Stop hosts at their next invocation; a running invocation still ends or times out. +fn cancel_hosts(cancel_marker: &Path) { + let _touched: std::io::Result<()> = touch(cancel_marker); } /// One host run decoded, or the message `RulesError` reports for it. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1af14e21b3..da5760b90e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -16258,10 +16258,12 @@ class CustomRuleRun(NativeRecord): ) -> None: ... class CustomHostPartition(NativeRecord): - """How one host of a split run detects module state and learns the run was cancelled.""" + """How a host learns of a cancelled run and whether it detects module state.""" cancelled: Callable[[], bool] + detect_state: bool def __init__( self, cancelled: Callable[[], bool], + detect_state: bool = ..., ) -> None: ... diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 132ae77ba9..c0349471bf 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -99,7 +99,7 @@ def evaluate_custom_rules( reads=reads, rules=custom_rules, rules_root=project_dir / "rules", - detect_state=partition is not None, + detect_state=partition is not None and partition.detect_state, ) inputs: _PassInputs = _PassInputs( diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 4b1dad7dd5..15246c448b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -67,6 +67,7 @@ def main() -> int: project_dir: Path = Path(str(payload["project_dir"])).resolve() dialect: str = str(payload.get("dialect", "generic")) verify_determinism: bool = payload.get("verify_determinism") is True + detect_state: bool = payload.get("detect_state") is True track_reads: bool = payload.get("track_reads") is True plan: CustomRulePlan = _decode_plan(payload["plan"]) os.environ.clear() @@ -95,7 +96,9 @@ def main() -> int: partition=( None if cancel_marker is None - else CustomHostPartition(cancelled=cancel_marker.exists) + else CustomHostPartition( + cancelled=cancel_marker.exists, detect_state=detect_state + ) ), ) except HostCancelledError: diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py index c18bbe10dc..66545c1f89 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py @@ -51,6 +51,20 @@ class HostInterruptTestCase: failing: tuple[str, ...] = () +@dataclass(frozen=True) +class SingleHostInterruptTestCase: + """An interrupt sent once `interrupt_after` model invocations have started on one host.""" + + description: str + model_count: int + hosts: int + delay_seconds: float + interrupt_after: int + expected_host_launches: int + expected_max_started_after_interrupt: int + failing: tuple[str, ...] = () + + @dataclass(frozen=True) class ModuleStateCase: """Custom-rule modules whose state changes both detectors must classify identically.""" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 6569d66040..4465ba467f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -22,6 +22,7 @@ HostInputsTestCase, HostInterruptTestCase, HostPartitionTestCase, + SingleHostInterruptTestCase, ) from tests.unit.src.sqlbuild.rule_engine._helpers.host.helpers import ( child_processes, @@ -316,6 +317,76 @@ def test_given_interrupt_while_hosts_are_busy_when_the_run_ends_then_the_interru assert outcomes[-1:] == ["KeyboardInterrupt"] +@pytest.mark.parametrize( + "test_case", + [ + SingleHostInterruptTestCase( + description="one host stops after the invocation in flight", + model_count=12, + hosts=1, + delay_seconds=0.3, + interrupt_after=3, + expected_host_launches=1, + expected_max_started_after_interrupt=1, + ), + SingleHostInterruptTestCase( + description="the single-host fallback stops after the invocation in flight", + model_count=30, + hosts=3, + delay_seconds=0.2, + interrupt_after=33, + expected_host_launches=4, + expected_max_started_after_interrupt=1, + failing=("orders_029",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_single_host_run_when_interrupted_then_it_stops_after_the_invocation_in_flight( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: SingleHostInterruptTestCase +) -> None: + started: Path = tmp_path / "started" + started.mkdir() + write_order_rules( + root=tmp_path, + delay_seconds=test_case.delay_seconds, + started_dir=started, + failing=test_case.failing, + ) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) + children_before: frozenset[int] = child_processes() + hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) + started_at_interrupt: list[int] = [] + + def ready() -> bool: + started_at_interrupt[:] = [len(tuple(started.iterdir()))] + return started_at_interrupt[0] >= test_case.interrupt_after + + outcomes: list[str] = [] + + with default_interrupt_handling(): + try: + with interrupt_when(ready) as sent: + try: + _ = evaluate_order_rules(project=project, root=tmp_path, cache_enabled=False) + except RulesError: + outcomes.append("RulesError") + for _ in range(100): + time.sleep(0.01) # a pending interrupt is raised between bytecodes + except KeyboardInterrupt: + outcomes.append("KeyboardInterrupt") + + assert sent.is_set() + assert outcomes == ["KeyboardInterrupt"] + assert len(hosts.launches()) == test_case.expected_host_launches + assert ( + len(tuple(started.iterdir())) - started_at_interrupt[0] + <= test_case.expected_max_started_after_interrupt + ) + assert child_processes() <= children_before + assert not host_inputs_left(tmp_path) + + @pytest.mark.parametrize( "test_case", (