Skip to content

Commit fce05cd

Browse files
authored
fix: stop a single custom-rule host at its next invocation when interrupted (#1535)
Ctrl-C during custom rules that run on one host, the default for small projects, waited until that host had run every invocation. Only split hosts had a cancellation marker; a single-host run, the single-host fallback after a failed split and the stateful rerun had none, and the fallback did not check for interrupts at all. Every host run now gets its own cancellation marker and waits for its host the way split hosts do, so an interrupt stops the host after the invocation in flight. Module-state detection, which the marker used to switch on, is now its own flag, still set only for split hosts.
1 parent 02d3c9d commit fce05cd

7 files changed

Lines changed: 158 additions & 45 deletions

File tree

‎crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,8 +367,11 @@ const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec {
367367
};
368368
const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec {
369369
name: "CustomHostPartition",
370-
doc: "How one host of a split run detects module state and learns the run was cancelled.",
371-
fields: &[FieldSpec::required("cancelled")],
370+
doc: "How a host learns of a cancelled run and whether it detects module state.",
371+
fields: &[
372+
FieldSpec::required("cancelled"),
373+
FieldSpec::new("detect_state", FieldDefault::False),
374+
],
372375
};
373376

374377
/// The rule engine value records.

‎crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs‎

Lines changed: 60 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -164,13 +164,12 @@ impl Launch {
164164
host_outcome(spec)
165165
}
166166

167-
/// The `run_custom_host_json` request for one host's plan and optional cancellation marker.
168-
fn spec(&self, plan: &HostPlan, cancel_marker: Option<&Path>) -> String {
167+
/// The host request for one plan and cancel marker; only split hosts detect module state.
168+
fn spec(&self, plan: &HostPlan, cancel_marker: &Path, detect_state: bool) -> String {
169169
let mut payload: Map<String, Value> = self.payload.clone();
170170
payload.insert("plan".to_owned(), json!(plan));
171-
if let Some(marker) = cancel_marker {
172-
payload.insert("cancel_marker".to_owned(), json!(text(marker)));
173-
}
171+
payload.insert("cancel_marker".to_owned(), json!(text(cancel_marker)));
172+
payload.insert("detect_state".to_owned(), json!(detect_state));
174173
json!({
175174
"program": self.program,
176175
"arguments": self.arguments,
@@ -189,36 +188,23 @@ fn run_partitions<'py>(
189188
(input_dir, cancel): (&Path, &HostCancel),
190189
(write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path),
191190
) -> PyResult<Vec<HostSlice>> {
192-
if plans.len() == 1 {
193-
let spec: String = launch.spec(plan, None);
194-
let outcomes: Vec<HostOutcome> = run_hosts(
195-
py,
196-
(launch, vec![spec]),
197-
(write_inputs, abandoned),
198-
(None, cancel),
199-
)?;
191+
let split: bool = plans.len() > 1;
192+
let outcomes: Vec<HostOutcome> = run_cancellable(
193+
py,
194+
(launch, plans, split),
195+
(input_dir, cancel),
196+
(write_inputs, abandoned),
197+
)?;
198+
if !split {
200199
return outcomes
201200
.into_iter()
202201
.map(|outcome| outcome.map_err(|message| rules_error(py, message)))
203202
.collect();
204203
}
205-
let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?;
206-
let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER);
207-
let specs: Vec<String> = plans
208-
.iter()
209-
.map(|partition| launch.spec(partition, Some(&marker)))
210-
.collect();
211-
let outcomes: PyResult<Vec<HostOutcome>> = run_hosts(
212-
py,
213-
(launch, specs),
214-
(write_inputs, abandoned),
215-
(Some(&marker), cancel),
216-
);
217-
let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir);
218-
let outcomes: Vec<HostOutcome> = outcomes?;
219204
interrupted(py, cancel)?;
220205
if outcomes.iter().any(Result::is_err) {
221-
return single_host(py, launch, plan).map(|slice| vec![slice]);
206+
return single_host(py, (launch, plan), (input_dir, cancel), abandoned)
207+
.map(|slice| vec![slice]);
222208
}
223209
let slices: Vec<HostSlice> = outcomes.into_iter().filter_map(Result::ok).collect();
224210
let stateful: BTreeSet<String> = slices
@@ -239,7 +225,7 @@ fn run_partitions<'py>(
239225
(code.clone(), subjects)
240226
})
241227
.collect();
242-
let rerun: HostSlice = single_host(py, launch, &rerun_plan)?;
228+
let rerun: HostSlice = single_host(py, (launch, &rerun_plan), (input_dir, cancel), abandoned)?;
243229
let mut merged: Vec<HostSlice> = slices
244230
.into_iter()
245231
.map(|slice| without_codes(slice, &stateful))
@@ -248,19 +234,55 @@ fn run_partitions<'py>(
248234
Ok(merged)
249235
}
250236

251-
/// Run one host for `plan` once the payload already exists.
252-
fn single_host(py: Python<'_>, launch: &Launch, plan: &HostPlan) -> PyResult<HostSlice> {
253-
let spec: String = launch.spec(plan, None);
254-
py.detach(|| launch.run(&spec))
237+
/// Run one host for `plan` from its written payload; an interrupt stops its next invocation.
238+
fn single_host(
239+
py: Python<'_>,
240+
(launch, plan): (&Launch, &HostPlan),
241+
(input_dir, cancel): (&Path, &HostCancel),
242+
abandoned: &Path,
243+
) -> PyResult<HostSlice> {
244+
let outcomes: Vec<HostOutcome> = run_cancellable(
245+
py,
246+
(launch, std::slice::from_ref(plan), false),
247+
(input_dir, cancel),
248+
(|| Ok(()), abandoned),
249+
)?;
250+
outcomes
251+
.into_iter()
252+
.next()
253+
.unwrap_or_else(|| Err("custom rule host returned no outcome".to_owned()))
255254
.map_err(|message| rules_error(py, message))
256255
}
257256

257+
/// Run one host per plan with a fresh cancel marker that an interrupt or failed host sets.
258+
fn run_cancellable(
259+
py: Python<'_>,
260+
(launch, plans, detect_state): (&Launch, &[HostPlan], bool),
261+
(input_dir, cancel): (&Path, &HostCancel),
262+
(write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path),
263+
) -> PyResult<Vec<HostOutcome>> {
264+
let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?;
265+
let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER);
266+
let specs: Vec<String> = plans
267+
.iter()
268+
.map(|partition| launch.spec(partition, &marker, detect_state))
269+
.collect();
270+
let outcomes: PyResult<Vec<HostOutcome>> = run_hosts(
271+
py,
272+
(launch, specs),
273+
(write_inputs, abandoned),
274+
(&marker, cancel),
275+
);
276+
let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir);
277+
outcomes
278+
}
279+
258280
/// Run every host on its own thread while this thread publishes the payload they wait for.
259281
fn run_hosts(
260282
py: Python<'_>,
261283
(launch, specs): (&Launch, Vec<String>),
262284
(write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path),
263-
(cancel_marker, cancel): (Option<&Path>, &HostCancel),
285+
(cancel_marker, cancel): (&Path, &HostCancel),
264286
) -> PyResult<Vec<HostOutcome>> {
265287
std::thread::scope(|scope| {
266288
let (sender, finished): (Sender<HostStatus>, Receiver<HostStatus>) = channel();
@@ -305,7 +327,7 @@ fn run_hosts(
305327
fn await_hosts(
306328
py: Python<'_>,
307329
(finished, count): (Receiver<HostStatus>, usize),
308-
cancel_marker: Option<&Path>,
330+
cancel_marker: &Path,
309331
cancel: &HostCancel,
310332
) -> PyResult<()> {
311333
let finished: Mutex<Receiver<HostStatus>> = Mutex::new(finished);
@@ -341,11 +363,9 @@ fn interrupted(py: Python<'_>, cancel: &HostCancel) -> PyResult<()> {
341363
Ok(())
342364
}
343365

344-
/// Stop split hosts at their next invocation; a running invocation still ends or times out.
345-
fn cancel_hosts(cancel_marker: Option<&Path>) {
346-
if let Some(marker) = cancel_marker {
347-
let _touched: std::io::Result<()> = touch(marker);
348-
}
366+
/// Stop hosts at their next invocation; a running invocation still ends or times out.
367+
fn cancel_hosts(cancel_marker: &Path) {
368+
let _touched: std::io::Result<()> = touch(cancel_marker);
349369
}
350370

351371
/// One host run decoded, or the message `RulesError` reports for it.

‎src/sqlbuild/_native.pyi‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16258,10 +16258,12 @@ class CustomRuleRun(NativeRecord):
1625816258
) -> None: ...
1625916259

1626016260
class CustomHostPartition(NativeRecord):
16261-
"""How one host of a split run detects module state and learns the run was cancelled."""
16261+
"""How a host learns of a cancelled run and whether it detects module state."""
1626216262

1626316263
cancelled: Callable[[], bool]
16264+
detect_state: bool
1626416265
def __init__(
1626516266
self,
1626616267
cancelled: Callable[[], bool],
16268+
detect_state: bool = ...,
1626716269
) -> None: ...

‎src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ def evaluate_custom_rules(
9999
reads=reads,
100100
rules=custom_rules,
101101
rules_root=project_dir / "rules",
102-
detect_state=partition is not None,
102+
detect_state=partition is not None and partition.detect_state,
103103
)
104104

105105
inputs: _PassInputs = _PassInputs(

‎src/sqlbuild/rule_engine/_helpers/host/custom_host.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ def main() -> int:
6767
project_dir: Path = Path(str(payload["project_dir"])).resolve()
6868
dialect: str = str(payload.get("dialect", "generic"))
6969
verify_determinism: bool = payload.get("verify_determinism") is True
70+
detect_state: bool = payload.get("detect_state") is True
7071
track_reads: bool = payload.get("track_reads") is True
7172
plan: CustomRulePlan = _decode_plan(payload["plan"])
7273
os.environ.clear()
@@ -95,7 +96,9 @@ def main() -> int:
9596
partition=(
9697
None
9798
if cancel_marker is None
98-
else CustomHostPartition(cancelled=cancel_marker.exists)
99+
else CustomHostPartition(
100+
cancelled=cancel_marker.exists, detect_state=detect_state
101+
)
99102
),
100103
)
101104
except HostCancelledError:

‎tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,20 @@ class HostInterruptTestCase:
5151
failing: tuple[str, ...] = ()
5252

5353

54+
@dataclass(frozen=True)
55+
class SingleHostInterruptTestCase:
56+
"""An interrupt sent once `interrupt_after` model invocations have started on one host."""
57+
58+
description: str
59+
model_count: int
60+
hosts: int
61+
delay_seconds: float
62+
interrupt_after: int
63+
expected_host_launches: int
64+
expected_max_started_after_interrupt: int
65+
failing: tuple[str, ...] = ()
66+
67+
5468
@dataclass(frozen=True)
5569
class ModuleStateCase:
5670
"""Custom-rule modules whose state changes both detectors must classify identically."""

‎tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
HostInputsTestCase,
2323
HostInterruptTestCase,
2424
HostPartitionTestCase,
25+
SingleHostInterruptTestCase,
2526
)
2627
from tests.unit.src.sqlbuild.rule_engine._helpers.host.helpers import (
2728
child_processes,
@@ -316,6 +317,76 @@ def test_given_interrupt_while_hosts_are_busy_when_the_run_ends_then_the_interru
316317
assert outcomes[-1:] == ["KeyboardInterrupt"]
317318

318319

320+
@pytest.mark.parametrize(
321+
"test_case",
322+
[
323+
SingleHostInterruptTestCase(
324+
description="one host stops after the invocation in flight",
325+
model_count=12,
326+
hosts=1,
327+
delay_seconds=0.3,
328+
interrupt_after=3,
329+
expected_host_launches=1,
330+
expected_max_started_after_interrupt=1,
331+
),
332+
SingleHostInterruptTestCase(
333+
description="the single-host fallback stops after the invocation in flight",
334+
model_count=30,
335+
hosts=3,
336+
delay_seconds=0.2,
337+
interrupt_after=33,
338+
expected_host_launches=4,
339+
expected_max_started_after_interrupt=1,
340+
failing=("orders_029",),
341+
),
342+
],
343+
ids=lambda case: case.description,
344+
)
345+
def test_given_a_single_host_run_when_interrupted_then_it_stops_after_the_invocation_in_flight(
346+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: SingleHostInterruptTestCase
347+
) -> None:
348+
started: Path = tmp_path / "started"
349+
started.mkdir()
350+
write_order_rules(
351+
root=tmp_path,
352+
delay_seconds=test_case.delay_seconds,
353+
started_dir=started,
354+
failing=test_case.failing,
355+
)
356+
project: CompiledProjectView = orders_project(model_count=test_case.model_count)
357+
children_before: frozenset[int] = child_processes()
358+
hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts)
359+
started_at_interrupt: list[int] = []
360+
361+
def ready() -> bool:
362+
started_at_interrupt[:] = [len(tuple(started.iterdir()))]
363+
return started_at_interrupt[0] >= test_case.interrupt_after
364+
365+
outcomes: list[str] = []
366+
367+
with default_interrupt_handling():
368+
try:
369+
with interrupt_when(ready) as sent:
370+
try:
371+
_ = evaluate_order_rules(project=project, root=tmp_path, cache_enabled=False)
372+
except RulesError:
373+
outcomes.append("RulesError")
374+
for _ in range(100):
375+
time.sleep(0.01) # a pending interrupt is raised between bytecodes
376+
except KeyboardInterrupt:
377+
outcomes.append("KeyboardInterrupt")
378+
379+
assert sent.is_set()
380+
assert outcomes == ["KeyboardInterrupt"]
381+
assert len(hosts.launches()) == test_case.expected_host_launches
382+
assert (
383+
len(tuple(started.iterdir())) - started_at_interrupt[0]
384+
<= test_case.expected_max_started_after_interrupt
385+
)
386+
assert child_processes() <= children_before
387+
assert not host_inputs_left(tmp_path)
388+
389+
319390
@pytest.mark.parametrize(
320391
"test_case",
321392
(

0 commit comments

Comments
 (0)