diff --git a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs index 989133a1cc..34f0f5b1bc 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/rule_records/constants.rs @@ -367,8 +367,11 @@ const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec { }; const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec { name: "CustomHostPartition", - doc: "How one host of a split run detects module state and learns the run was cancelled.", - fields: &[FieldSpec::required("cancelled")], + doc: "How a host learns of a cancelled run and whether it detects module state.", + fields: &[ + FieldSpec::required("cancelled"), + FieldSpec::new("detect_state", FieldDefault::False), + ], }; /// The rule engine value records. diff --git a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs index 532b3414d1..5849cad6a8 100644 --- a/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs +++ b/crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_hosts.rs @@ -164,13 +164,12 @@ impl Launch { host_outcome(spec) } - /// The `run_custom_host_json` request for one host's plan and optional cancellation marker. - fn spec(&self, plan: &HostPlan, cancel_marker: Option<&Path>) -> String { + /// The host request for one plan and cancel marker; only split hosts detect module state. + fn spec(&self, plan: &HostPlan, cancel_marker: &Path, detect_state: bool) -> String { let mut payload: Map = self.payload.clone(); payload.insert("plan".to_owned(), json!(plan)); - if let Some(marker) = cancel_marker { - payload.insert("cancel_marker".to_owned(), json!(text(marker))); - } + payload.insert("cancel_marker".to_owned(), json!(text(cancel_marker))); + payload.insert("detect_state".to_owned(), json!(detect_state)); json!({ "program": self.program, "arguments": self.arguments, @@ -189,36 +188,23 @@ fn run_partitions<'py>( (input_dir, cancel): (&Path, &HostCancel), (write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path), ) -> PyResult> { - if plans.len() == 1 { - let spec: String = launch.spec(plan, None); - let outcomes: Vec = run_hosts( - py, - (launch, vec![spec]), - (write_inputs, abandoned), - (None, cancel), - )?; + let split: bool = plans.len() > 1; + let outcomes: Vec = run_cancellable( + py, + (launch, plans, split), + (input_dir, cancel), + (write_inputs, abandoned), + )?; + if !split { return outcomes .into_iter() .map(|outcome| outcome.map_err(|message| rules_error(py, message))) .collect(); } - let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?; - let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER); - let specs: Vec = plans - .iter() - .map(|partition| launch.spec(partition, Some(&marker))) - .collect(); - let outcomes: PyResult> = run_hosts( - py, - (launch, specs), - (write_inputs, abandoned), - (Some(&marker), cancel), - ); - let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir); - let outcomes: Vec = outcomes?; interrupted(py, cancel)?; if outcomes.iter().any(Result::is_err) { - return single_host(py, launch, plan).map(|slice| vec![slice]); + return single_host(py, (launch, plan), (input_dir, cancel), abandoned) + .map(|slice| vec![slice]); } let slices: Vec = outcomes.into_iter().filter_map(Result::ok).collect(); let stateful: BTreeSet = slices @@ -239,7 +225,7 @@ fn run_partitions<'py>( (code.clone(), subjects) }) .collect(); - let rerun: HostSlice = single_host(py, launch, &rerun_plan)?; + let rerun: HostSlice = single_host(py, (launch, &rerun_plan), (input_dir, cancel), abandoned)?; let mut merged: Vec = slices .into_iter() .map(|slice| without_codes(slice, &stateful)) @@ -248,19 +234,55 @@ fn run_partitions<'py>( Ok(merged) } -/// Run one host for `plan` once the payload already exists. -fn single_host(py: Python<'_>, launch: &Launch, plan: &HostPlan) -> PyResult { - let spec: String = launch.spec(plan, None); - py.detach(|| launch.run(&spec)) +/// Run one host for `plan` from its written payload; an interrupt stops its next invocation. +fn single_host( + py: Python<'_>, + (launch, plan): (&Launch, &HostPlan), + (input_dir, cancel): (&Path, &HostCancel), + abandoned: &Path, +) -> PyResult { + let outcomes: Vec = run_cancellable( + py, + (launch, std::slice::from_ref(plan), false), + (input_dir, cancel), + (|| Ok(()), abandoned), + )?; + outcomes + .into_iter() + .next() + .unwrap_or_else(|| Err("custom rule host returned no outcome".to_owned())) .map_err(|message| rules_error(py, message)) } +/// Run one host per plan with a fresh cancel marker that an interrupt or failed host sets. +fn run_cancellable( + py: Python<'_>, + (launch, plans, detect_state): (&Launch, &[HostPlan], bool), + (input_dir, cancel): (&Path, &HostCancel), + (write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path), +) -> PyResult> { + let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?; + let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER); + let specs: Vec = plans + .iter() + .map(|partition| launch.spec(partition, &marker, detect_state)) + .collect(); + let outcomes: PyResult> = run_hosts( + py, + (launch, specs), + (write_inputs, abandoned), + (&marker, cancel), + ); + let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir); + outcomes +} + /// Run every host on its own thread while this thread publishes the payload they wait for. fn run_hosts( py: Python<'_>, (launch, specs): (&Launch, Vec), (write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path), - (cancel_marker, cancel): (Option<&Path>, &HostCancel), + (cancel_marker, cancel): (&Path, &HostCancel), ) -> PyResult> { std::thread::scope(|scope| { let (sender, finished): (Sender, Receiver) = channel(); @@ -305,7 +327,7 @@ fn run_hosts( fn await_hosts( py: Python<'_>, (finished, count): (Receiver, usize), - cancel_marker: Option<&Path>, + cancel_marker: &Path, cancel: &HostCancel, ) -> PyResult<()> { let finished: Mutex> = Mutex::new(finished); @@ -341,11 +363,9 @@ fn interrupted(py: Python<'_>, cancel: &HostCancel) -> PyResult<()> { Ok(()) } -/// Stop split hosts at their next invocation; a running invocation still ends or times out. -fn cancel_hosts(cancel_marker: Option<&Path>) { - if let Some(marker) = cancel_marker { - let _touched: std::io::Result<()> = touch(marker); - } +/// Stop hosts at their next invocation; a running invocation still ends or times out. +fn cancel_hosts(cancel_marker: &Path) { + let _touched: std::io::Result<()> = touch(cancel_marker); } /// One host run decoded, or the message `RulesError` reports for it. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1af14e21b3..da5760b90e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -16258,10 +16258,12 @@ class CustomRuleRun(NativeRecord): ) -> None: ... class CustomHostPartition(NativeRecord): - """How one host of a split run detects module state and learns the run was cancelled.""" + """How a host learns of a cancelled run and whether it detects module state.""" cancelled: Callable[[], bool] + detect_state: bool def __init__( self, cancelled: Callable[[], bool], + detect_state: bool = ..., ) -> None: ... diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 132ae77ba9..c0349471bf 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -99,7 +99,7 @@ def evaluate_custom_rules( reads=reads, rules=custom_rules, rules_root=project_dir / "rules", - detect_state=partition is not None, + detect_state=partition is not None and partition.detect_state, ) inputs: _PassInputs = _PassInputs( diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 4b1dad7dd5..15246c448b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -67,6 +67,7 @@ def main() -> int: project_dir: Path = Path(str(payload["project_dir"])).resolve() dialect: str = str(payload.get("dialect", "generic")) verify_determinism: bool = payload.get("verify_determinism") is True + detect_state: bool = payload.get("detect_state") is True track_reads: bool = payload.get("track_reads") is True plan: CustomRulePlan = _decode_plan(payload["plan"]) os.environ.clear() @@ -95,7 +96,9 @@ def main() -> int: partition=( None if cancel_marker is None - else CustomHostPartition(cancelled=cancel_marker.exists) + else CustomHostPartition( + cancelled=cancel_marker.exists, detect_state=detect_state + ) ), ) except HostCancelledError: diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py index c18bbe10dc..66545c1f89 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py @@ -51,6 +51,20 @@ class HostInterruptTestCase: failing: tuple[str, ...] = () +@dataclass(frozen=True) +class SingleHostInterruptTestCase: + """An interrupt sent once `interrupt_after` model invocations have started on one host.""" + + description: str + model_count: int + hosts: int + delay_seconds: float + interrupt_after: int + expected_host_launches: int + expected_max_started_after_interrupt: int + failing: tuple[str, ...] = () + + @dataclass(frozen=True) class ModuleStateCase: """Custom-rule modules whose state changes both detectors must classify identically.""" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 6569d66040..4465ba467f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -22,6 +22,7 @@ HostInputsTestCase, HostInterruptTestCase, HostPartitionTestCase, + SingleHostInterruptTestCase, ) from tests.unit.src.sqlbuild.rule_engine._helpers.host.helpers import ( child_processes, @@ -316,6 +317,76 @@ def test_given_interrupt_while_hosts_are_busy_when_the_run_ends_then_the_interru assert outcomes[-1:] == ["KeyboardInterrupt"] +@pytest.mark.parametrize( + "test_case", + [ + SingleHostInterruptTestCase( + description="one host stops after the invocation in flight", + model_count=12, + hosts=1, + delay_seconds=0.3, + interrupt_after=3, + expected_host_launches=1, + expected_max_started_after_interrupt=1, + ), + SingleHostInterruptTestCase( + description="the single-host fallback stops after the invocation in flight", + model_count=30, + hosts=3, + delay_seconds=0.2, + interrupt_after=33, + expected_host_launches=4, + expected_max_started_after_interrupt=1, + failing=("orders_029",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_single_host_run_when_interrupted_then_it_stops_after_the_invocation_in_flight( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: SingleHostInterruptTestCase +) -> None: + started: Path = tmp_path / "started" + started.mkdir() + write_order_rules( + root=tmp_path, + delay_seconds=test_case.delay_seconds, + started_dir=started, + failing=test_case.failing, + ) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) + children_before: frozenset[int] = child_processes() + hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) + started_at_interrupt: list[int] = [] + + def ready() -> bool: + started_at_interrupt[:] = [len(tuple(started.iterdir()))] + return started_at_interrupt[0] >= test_case.interrupt_after + + outcomes: list[str] = [] + + with default_interrupt_handling(): + try: + with interrupt_when(ready) as sent: + try: + _ = evaluate_order_rules(project=project, root=tmp_path, cache_enabled=False) + except RulesError: + outcomes.append("RulesError") + for _ in range(100): + time.sleep(0.01) # a pending interrupt is raised between bytecodes + except KeyboardInterrupt: + outcomes.append("KeyboardInterrupt") + + assert sent.is_set() + assert outcomes == ["KeyboardInterrupt"] + assert len(hosts.launches()) == test_case.expected_host_launches + assert ( + len(tuple(started.iterdir())) - started_at_interrupt[0] + <= test_case.expected_max_started_after_interrupt + ) + assert child_processes() <= children_before + assert not host_inputs_left(tmp_path) + + @pytest.mark.parametrize( "test_case", (