Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -367,8 +367,11 @@ const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec {
};
const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec {
name: "CustomHostPartition",
doc: "How one host of a split run detects module state and learns the run was cancelled.",
fields: &[FieldSpec::required("cancelled")],
doc: "How a host learns of a cancelled run and whether it detects module state.",
fields: &[
FieldSpec::required("cancelled"),
FieldSpec::new("detect_state", FieldDefault::False),
],
};

/// The rule engine value records.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,13 +164,12 @@ impl Launch {
host_outcome(spec)
}

/// The `run_custom_host_json` request for one host's plan and optional cancellation marker.
fn spec(&self, plan: &HostPlan, cancel_marker: Option<&Path>) -> String {
/// The host request for one plan and cancel marker; only split hosts detect module state.
fn spec(&self, plan: &HostPlan, cancel_marker: &Path, detect_state: bool) -> String {
let mut payload: Map<String, Value> = self.payload.clone();
payload.insert("plan".to_owned(), json!(plan));
if let Some(marker) = cancel_marker {
payload.insert("cancel_marker".to_owned(), json!(text(marker)));
}
payload.insert("cancel_marker".to_owned(), json!(text(cancel_marker)));
payload.insert("detect_state".to_owned(), json!(detect_state));
json!({
"program": self.program,
"arguments": self.arguments,
Expand All @@ -189,36 +188,23 @@ fn run_partitions<'py>(
(input_dir, cancel): (&Path, &HostCancel),
(write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path),
) -> PyResult<Vec<HostSlice>> {
if plans.len() == 1 {
let spec: String = launch.spec(plan, None);
let outcomes: Vec<HostOutcome> = run_hosts(
py,
(launch, vec![spec]),
(write_inputs, abandoned),
(None, cancel),
)?;
let split: bool = plans.len() > 1;
let outcomes: Vec<HostOutcome> = run_cancellable(
py,
(launch, plans, split),
(input_dir, cancel),
(write_inputs, abandoned),
)?;
if !split {
return outcomes
.into_iter()
.map(|outcome| outcome.map_err(|message| rules_error(py, message)))
.collect();
}
let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?;
let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER);
let specs: Vec<String> = plans
.iter()
.map(|partition| launch.spec(partition, Some(&marker)))
.collect();
let outcomes: PyResult<Vec<HostOutcome>> = run_hosts(
py,
(launch, specs),
(write_inputs, abandoned),
(Some(&marker), cancel),
);
let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir);
let outcomes: Vec<HostOutcome> = outcomes?;
interrupted(py, cancel)?;
if outcomes.iter().any(Result::is_err) {
return single_host(py, launch, plan).map(|slice| vec![slice]);
return single_host(py, (launch, plan), (input_dir, cancel), abandoned)
.map(|slice| vec![slice]);
}
let slices: Vec<HostSlice> = outcomes.into_iter().filter_map(Result::ok).collect();
let stateful: BTreeSet<String> = slices
Expand All @@ -239,7 +225,7 @@ fn run_partitions<'py>(
(code.clone(), subjects)
})
.collect();
let rerun: HostSlice = single_host(py, launch, &rerun_plan)?;
let rerun: HostSlice = single_host(py, (launch, &rerun_plan), (input_dir, cancel), abandoned)?;
let mut merged: Vec<HostSlice> = slices
.into_iter()
.map(|slice| without_codes(slice, &stateful))
Expand All @@ -248,19 +234,55 @@ fn run_partitions<'py>(
Ok(merged)
}

/// Run one host for `plan` once the payload already exists.
fn single_host(py: Python<'_>, launch: &Launch, plan: &HostPlan) -> PyResult<HostSlice> {
let spec: String = launch.spec(plan, None);
py.detach(|| launch.run(&spec))
/// Run one host for `plan` from its written payload; an interrupt stops its next invocation.
fn single_host(
py: Python<'_>,
(launch, plan): (&Launch, &HostPlan),
(input_dir, cancel): (&Path, &HostCancel),
abandoned: &Path,
) -> PyResult<HostSlice> {
let outcomes: Vec<HostOutcome> = run_cancellable(
py,
(launch, std::slice::from_ref(plan), false),
(input_dir, cancel),
(|| Ok(()), abandoned),
)?;
outcomes
.into_iter()
.next()
.unwrap_or_else(|| Err("custom rule host returned no outcome".to_owned()))
.map_err(|message| rules_error(py, message))
}

/// Run one host per plan with a fresh cancel marker that an interrupt or failed host sets.
fn run_cancellable(
py: Python<'_>,
(launch, plans, detect_state): (&Launch, &[HostPlan], bool),
(input_dir, cancel): (&Path, &HostCancel),
(write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path),
) -> PyResult<Vec<HostOutcome>> {
let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?;
let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER);
let specs: Vec<String> = plans
.iter()
.map(|partition| launch.spec(partition, &marker, detect_state))
.collect();
let outcomes: PyResult<Vec<HostOutcome>> = run_hosts(
py,
(launch, specs),
(write_inputs, abandoned),
(&marker, cancel),
);
let _removed: std::io::Result<()> = std::fs::remove_dir_all(&cancel_dir);
outcomes
}

/// Run every host on its own thread while this thread publishes the payload they wait for.
fn run_hosts(
py: Python<'_>,
(launch, specs): (&Launch, Vec<String>),
(write_inputs, abandoned): (impl FnOnce() -> PyResult<()>, &Path),
(cancel_marker, cancel): (Option<&Path>, &HostCancel),
(cancel_marker, cancel): (&Path, &HostCancel),
) -> PyResult<Vec<HostOutcome>> {
std::thread::scope(|scope| {
let (sender, finished): (Sender<HostStatus>, Receiver<HostStatus>) = channel();
Expand Down Expand Up @@ -305,7 +327,7 @@ fn run_hosts(
fn await_hosts(
py: Python<'_>,
(finished, count): (Receiver<HostStatus>, usize),
cancel_marker: Option<&Path>,
cancel_marker: &Path,
cancel: &HostCancel,
) -> PyResult<()> {
let finished: Mutex<Receiver<HostStatus>> = Mutex::new(finished);
Expand Down Expand Up @@ -341,11 +363,9 @@ fn interrupted(py: Python<'_>, cancel: &HostCancel) -> PyResult<()> {
Ok(())
}

/// Stop split hosts at their next invocation; a running invocation still ends or times out.
fn cancel_hosts(cancel_marker: Option<&Path>) {
if let Some(marker) = cancel_marker {
let _touched: std::io::Result<()> = touch(marker);
}
/// Stop hosts at their next invocation; a running invocation still ends or times out.
fn cancel_hosts(cancel_marker: &Path) {
let _touched: std::io::Result<()> = touch(cancel_marker);
}

/// One host run decoded, or the message `RulesError` reports for it.
Expand Down
4 changes: 3 additions & 1 deletion src/sqlbuild/_native.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -16258,10 +16258,12 @@ class CustomRuleRun(NativeRecord):
) -> None: ...

class CustomHostPartition(NativeRecord):
"""How one host of a split run detects module state and learns the run was cancelled."""
"""How a host learns of a cancelled run and whether it detects module state."""

cancelled: Callable[[], bool]
detect_state: bool
def __init__(
self,
cancelled: Callable[[], bool],
detect_state: bool = ...,
) -> None: ...
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def evaluate_custom_rules(
reads=reads,
rules=custom_rules,
rules_root=project_dir / "rules",
detect_state=partition is not None,
detect_state=partition is not None and partition.detect_state,
)

inputs: _PassInputs = _PassInputs(
Expand Down
5 changes: 4 additions & 1 deletion src/sqlbuild/rule_engine/_helpers/host/custom_host.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ def main() -> int:
project_dir: Path = Path(str(payload["project_dir"])).resolve()
dialect: str = str(payload.get("dialect", "generic"))
verify_determinism: bool = payload.get("verify_determinism") is True
detect_state: bool = payload.get("detect_state") is True
track_reads: bool = payload.get("track_reads") is True
plan: CustomRulePlan = _decode_plan(payload["plan"])
os.environ.clear()
Expand Down Expand Up @@ -95,7 +96,9 @@ def main() -> int:
partition=(
None
if cancel_marker is None
else CustomHostPartition(cancelled=cancel_marker.exists)
else CustomHostPartition(
cancelled=cancel_marker.exists, detect_state=detect_state
)
),
)
except HostCancelledError:
Expand Down
14 changes: 14 additions & 0 deletions tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,20 @@ class HostInterruptTestCase:
failing: tuple[str, ...] = ()


@dataclass(frozen=True)
class SingleHostInterruptTestCase:
"""An interrupt sent once `interrupt_after` model invocations have started on one host."""

description: str
model_count: int
hosts: int
delay_seconds: float
interrupt_after: int
expected_host_launches: int
expected_max_started_after_interrupt: int
failing: tuple[str, ...] = ()


@dataclass(frozen=True)
class ModuleStateCase:
"""Custom-rule modules whose state changes both detectors must classify identically."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
HostInputsTestCase,
HostInterruptTestCase,
HostPartitionTestCase,
SingleHostInterruptTestCase,
)
from tests.unit.src.sqlbuild.rule_engine._helpers.host.helpers import (
child_processes,
Expand Down Expand Up @@ -316,6 +317,76 @@ def test_given_interrupt_while_hosts_are_busy_when_the_run_ends_then_the_interru
assert outcomes[-1:] == ["KeyboardInterrupt"]


@pytest.mark.parametrize(
"test_case",
[
SingleHostInterruptTestCase(
description="one host stops after the invocation in flight",
model_count=12,
hosts=1,
delay_seconds=0.3,
interrupt_after=3,
expected_host_launches=1,
expected_max_started_after_interrupt=1,
),
SingleHostInterruptTestCase(
description="the single-host fallback stops after the invocation in flight",
model_count=30,
hosts=3,
delay_seconds=0.2,
interrupt_after=33,
expected_host_launches=4,
expected_max_started_after_interrupt=1,
failing=("orders_029",),
),
],
ids=lambda case: case.description,
)
def test_given_a_single_host_run_when_interrupted_then_it_stops_after_the_invocation_in_flight(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: SingleHostInterruptTestCase
) -> None:
started: Path = tmp_path / "started"
started.mkdir()
write_order_rules(
root=tmp_path,
delay_seconds=test_case.delay_seconds,
started_dir=started,
failing=test_case.failing,
)
project: CompiledProjectView = orders_project(model_count=test_case.model_count)
children_before: frozenset[int] = child_processes()
hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts)
started_at_interrupt: list[int] = []

def ready() -> bool:
started_at_interrupt[:] = [len(tuple(started.iterdir()))]
return started_at_interrupt[0] >= test_case.interrupt_after

outcomes: list[str] = []

with default_interrupt_handling():
try:
with interrupt_when(ready) as sent:
try:
_ = evaluate_order_rules(project=project, root=tmp_path, cache_enabled=False)
except RulesError:
outcomes.append("RulesError")
for _ in range(100):
time.sleep(0.01) # a pending interrupt is raised between bytecodes
except KeyboardInterrupt:
outcomes.append("KeyboardInterrupt")

assert sent.is_set()
assert outcomes == ["KeyboardInterrupt"]
assert len(hosts.launches()) == test_case.expected_host_launches
assert (
len(tuple(started.iterdir())) - started_at_interrupt[0]
<= test_case.expected_max_started_after_interrupt
)
assert child_processes() <= children_before
assert not host_inputs_left(tmp_path)


@pytest.mark.parametrize(
"test_case",
(
Expand Down
Loading