Skip to content

Commit f113542

Browse files
authored
fix: stop on SIGTERM when started with SIGINT ignored, and event-driven interrupt tests (#1531)
* test: drive the interrupt and SIGTERM tests by events, not wall-clock time The busy-split-hosts interrupt tests and four Recall SIGTERM and Ctrl-C tests sent their signal on a timer and asserted elapsed time, so they failed in runners that start commands with SIGINT ignored or under load. They now send the signal once the blocking step has started (rule invocations started, the task printed, the store shows the step running) and assert outcomes and counts: the interrupt was raised, hosts launched, fewer invocations than models, no child processes or host inputs left, the exit code, no statement without an end, and that the step's post-sleep output is absent. A generous bound remains only to fail a hang. Processes the tests interrupt get default SIGINT handling. A new unit test interrupts the single-host fallback and fails if the pending interrupt is ever lost. * fix: stop on SIGTERM when the process was started with SIGINT ignored After SIGTERM, the command cancels its running statements and interrupts its main thread by sending it SIGINT, but only when Python handles SIGINT. A process started with SIGINT ignored, as background jobs of scripts and many process managers and CI runners are, therefore kept running its blocking step, such as a task or a slow sink import, to the end. Orchestrators then SIGKILL it after their grace period, before it cleans up. When SIGINT is not handled, the command now installs a handler on an otherwise unused signal (SIGRTMIN, or SIGUSR2 where there is none) while that signal is at its default action, and the SIGTERM watcher interrupts the main thread with it. The handler raises the same termination error, so the command exits 143 as before, and is restored when the command ends. SIGINT itself is unchanged: when it was ignored, Ctrl-C stays ignored. Windows is unchanged. An end-to-end test starts the native sqb build with SIGINT ignored and sends SIGTERM once its task is running.
1 parent bcaf3cc commit f113542

17 files changed

Lines changed: 425 additions & 138 deletions

File tree

‎crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use std::cell::Cell;
44

55
use pyo3::exceptions::PyKeyboardInterrupt;
6-
use pyo3::prelude::{PyResult, Python};
6+
use pyo3::prelude::{Py, PyAny, PyAnyMethods, PyModule, PyResult, Python};
77

88
thread_local! {
99
pub(crate) static INTERRUPTED: Cell<bool> = const { Cell::new(false) };
@@ -23,3 +23,19 @@ pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> {
2323
}
2424
Ok(())
2525
}
26+
27+
/// Put `signal`'s previous Python handler back, unless something replaced `installed` meanwhile. Fails when a pending handler raised first, as `signal.signal` runs pending handlers before it changes one.
28+
pub(crate) fn restore_signal_handler(
29+
py: Python<'_>,
30+
signal: i64,
31+
(installed, previous): (&Py<PyAny>, &Py<PyAny>),
32+
) -> PyResult<()> {
33+
let module: pyo3::Bound<'_, PyModule> = py.import("signal")?;
34+
if module
35+
.call_method1("getsignal", (signal,))?
36+
.is(installed.bind(py))
37+
{
38+
module.call_method1("signal", (signal, previous.bind(py)))?;
39+
}
40+
Ok(())
41+
}

‎crates/sqlbuild-python-base/src/native_host/models.rs‎

Lines changed: 74 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@
22
33
use crate::native_host::_helpers::interrupt::INTERRUPTED;
44
use crate::native_host::_helpers::interrupt::SIGINT;
5+
use crate::native_host::_helpers::interrupt::restore_signal_handler;
56
use pyo3::exceptions::PyKeyboardInterrupt;
67
use pyo3::prelude::Bound;
78
use pyo3::prelude::Py;
89
use pyo3::prelude::PyAny;
910
use pyo3::prelude::PyAnyMethods;
11+
use pyo3::prelude::PyErr;
1012
use pyo3::prelude::PyResult;
1113
use pyo3::prelude::Python;
1214
use pyo3::sync::PyOnceLock;
@@ -107,22 +109,83 @@ impl InterruptHandler {
107109
impl Drop for InterruptHandler {
108110
fn drop(&mut self) {
109111
Python::attach(|py| {
110-
let restored: PyResult<()> = (|| {
111-
let signal = py.import("signal")?;
112-
if signal
113-
.call_method1("getsignal", (SIGINT,))?
114-
.is(self.installed.bind(py))
115-
{
116-
signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?;
117-
}
118-
Ok(())
119-
})();
120-
drop(restored);
112+
drop(restore_signal_handler(
113+
py,
114+
SIGINT,
115+
(&self.installed, &self.previous),
116+
));
121117
});
122118
INTERRUPTED.set(false);
123119
}
124120
}
125121

122+
/// A command's handler on an otherwise unused signal that raises the command's error, so another thread can interrupt the main thread while SIGINT is ignored, without changing how SIGINT is handled.
123+
#[derive(Debug)]
124+
pub struct WakeHandler {
125+
pub signal: i64,
126+
pub(crate) previous: Py<PyAny>,
127+
pub(crate) installed: Py<PyAny>,
128+
}
129+
130+
impl WakeHandler {
131+
/// The signal that interrupts the main thread: SIGINT where Python handles it, else the signal of a wake handler raising `error`, installed here; `None` when neither is available.
132+
pub fn interrupting_signal(
133+
py: Python<'_>,
134+
error: fn() -> PyErr,
135+
) -> PyResult<(Option<i64>, Option<Self>)> {
136+
let signal = py.import("signal")?;
137+
if signal.call_method1("getsignal", (SIGINT,))?.is_callable() {
138+
return Ok((Some(SIGINT), None));
139+
}
140+
let wake: Option<Self> = Self::install(py, error)?;
141+
Ok((wake.as_ref().map(|wake| wake.signal), wake))
142+
}
143+
144+
/// Install the handler on SIGRTMIN, or SIGUSR2 where there is none, while that signal takes its default action.
145+
fn install(py: Python<'_>, error: fn() -> PyErr) -> PyResult<Option<Self>> {
146+
let signal = py.import("signal")?;
147+
let name: &str = if signal.hasattr("SIGRTMIN")? {
148+
"SIGRTMIN"
149+
} else if signal.hasattr("SIGUSR2")? {
150+
"SIGUSR2"
151+
} else {
152+
return Ok(None);
153+
};
154+
let number: i64 = signal.getattr(name)?.extract()?;
155+
let previous = signal.call_method1("getsignal", (number,))?;
156+
if !previous.is(signal.getattr("SIG_DFL")?) {
157+
return Ok(None);
158+
}
159+
let installed = PyCFunction::new_closure(
160+
py,
161+
None,
162+
None,
163+
move |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> {
164+
Err(error())
165+
},
166+
)?;
167+
signal.call_method1("signal", (number, &installed))?;
168+
Ok(Some(Self {
169+
signal: number,
170+
previous: previous.unbind(),
171+
installed: installed.into_any().unbind(),
172+
}))
173+
}
174+
}
175+
176+
impl Drop for WakeHandler {
177+
/// Restore the previous handler. A wake still pending raises on the first attempt and is consumed, so the second attempt restores.
178+
fn drop(&mut self) {
179+
Python::attach(|py| {
180+
let handlers = (&self.installed, &self.previous);
181+
drop(
182+
restore_signal_handler(py, self.signal, handlers)
183+
.or_else(|_pending| restore_signal_handler(py, self.signal, handlers)),
184+
);
185+
});
186+
}
187+
}
188+
126189
impl std::fmt::Debug for AuditProjection {
127190
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
128191
formatter

‎crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use sqlbuild_host::entries::models::ExecutorHost;
1616
use crate::bindings::_helpers::recall::console::write_stderr;
1717
use sqlbuild_python_base::boundary::main::imported::imported;
1818
use sqlbuild_python_base::native_host::main::host_entry::host_entry;
19+
use sqlbuild_python_base::native_host::models::WakeHandler;
1920
use sqlbuild_python_base::records::main::fields::field::field as native_field;
2021

2122
create_exception!(
@@ -69,11 +70,12 @@ fn registered_flags() -> Option<&'static SigtermFlags> {
6970
.as_ref()
7071
}
7172

72-
/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. Holds only Rust values: while it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again.
73+
/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. While it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again.
7374
pub(crate) struct TerminationHandler {
7475
flags: Option<&'static SigtermFlags>,
7576
stop: Arc<AtomicBool>,
7677
watcher: Option<JoinHandle<()>>,
78+
wake: Option<WakeHandler>,
7779
}
7880

7981
impl TerminationHandler {
@@ -86,6 +88,7 @@ impl TerminationHandler {
8688
if let Some(watcher) = self.watcher.take() {
8789
py.detach(|| drop(watcher.join()));
8890
}
91+
drop(self.wake.take());
8992
terminated()
9093
}
9194
}
@@ -96,18 +99,19 @@ impl Drop for TerminationHandler {
9699
if let Some(watcher) = self.watcher.take() {
97100
Python::attach(|py| py.detach(|| drop(watcher.join())));
98101
}
102+
drop(self.wake.take());
99103
if let Some(flags) = self.flags {
100104
flags.default.store(true, Ordering::SeqCst);
101105
flags.terminated.store(false, Ordering::SeqCst);
102106
}
103107
}
104108
}
105109

106-
/// Where the watcher sends the interrupt: the main thread, and whether Python handles Ctrl+C there.
110+
/// The signal the watcher sends the main thread: SIGINT where Python handles Ctrl+C, else the wake signal.
107111
#[derive(Clone, Copy)]
108112
struct MainThread {
109113
ident: u64,
110-
handles_interrupt: bool,
114+
signal: Option<i64>,
111115
}
112116

113117
/// Once SIGTERM arrives: let a second one end the process at once, cancel every running statement, and interrupt the main thread the way Ctrl+C does, where the platform can signal one thread.
@@ -119,9 +123,9 @@ fn watch(flags: &SigtermFlags, stop: &AtomicBool, main: MainThread) {
119123
let cancelled = host_entry(py, ExecutorHost::CancelRunningStatements)
120124
.and_then(|cancel| cancel.call0());
121125
drop(cancelled);
122-
if main.handles_interrupt {
126+
if let Some(signal) = main.signal {
123127
let interrupted = imported(py, "signal", "pthread_kill")
124-
.and_then(|kill| kill.call1((main.ident, SIGINT_NUMBER)));
128+
.and_then(|kill| kill.call1((main.ident, signal)));
125129
drop(interrupted);
126130
}
127131
});
@@ -148,11 +152,14 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult<TerminationHandler> {
148152
flags: None,
149153
stop,
150154
watcher: None,
155+
wake: None,
151156
});
152157
};
158+
let (signal, wake): (Option<i64>, Option<WakeHandler>) =
159+
WakeHandler::interrupting_signal(py, || CommandTerminated::new_err(TERMINATED_MESSAGE))?;
153160
let main = MainThread {
154161
ident: native_field(&main_thread, "ident")?.extract()?,
155-
handles_interrupt: action(SIGINT_NUMBER)?.is_callable(),
162+
signal,
156163
};
157164
flags.terminated.store(false, Ordering::SeqCst);
158165
let watched: Arc<AtomicBool> = Arc::clone(&stop);
@@ -164,6 +171,7 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult<TerminationHandler> {
164171
flags: Some(flags),
165172
stop,
166173
watcher: Some(watcher),
174+
wake,
167175
})
168176
}
169177

‎fensu.toml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1655,6 +1655,11 @@ rule = "FFT104"
16551655
path = "tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py"
16561656
reason = "The disposable HTTPS gateway and OIDC provider dispatch real HTTP protocol messages and authentication transports; assertions in the parametrized installed journey remain branch-free."
16571657

1658+
[[targets.python.rule_exceptions]]
1659+
rule = "FFT104"
1660+
path = "tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py"
1661+
reason = "The interrupter thread polls for the event that starts the interrupt and must not interrupt once its block has ended; the tests using it remain branch-free."
1662+
16581663
[targets.rust]
16591664
analyzer = "rust"
16601665
roots = ["crates"]
@@ -1670,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X
16701675

16711676
[[targets.rust.threshold_overrides]]
16721677
paths = ["crates/sqlbuild-python/Cargo.toml"]
1673-
thresholds = { max_crate_code_lines = 66756 }
1678+
thresholds = { max_crate_code_lines = 66764 }
16741679
reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out."
16751680

16761681
[[targets.rust.threshold_overrides]]

‎src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,11 @@ can contain sensitive SQL, output and audit evidence.
155155

156156
A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it
157157
cancels the statements it is running, records its real end time, its output and exit code 143,
158-
prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process
159-
at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run
160-
stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end:
158+
prints `error: terminated by SIGTERM`, and exits with code 143. It does this also when it was
159+
started with SIGINT ignored, as a script starts a background job and some process managers start a
160+
service; SIGINT and Ctrl-C then stay ignored. A second SIGTERM ends the process at once. Recall
161+
keeps heartbeating while the command discovers and compiles the project, so a run stopped in any
162+
phase is never mistaken for one that died. An `abandoned` run cannot record its end:
161163
its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`.
162164

163165
## Selection

‎tests/integration/src/sqlbuild/cli/commands/main/_test_types.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,7 @@ class CustomHostInterruptIntegrationTestCase:
215215
description: str
216216
model_count: int
217217
hosts: int
218-
interrupt_after_seconds: float
219-
expected_max_seconds: float
218+
expected_host_launches: int
220219

221220

222221
@dataclass(frozen=True)

‎tests/integration/src/sqlbuild/cli/commands/main/helpers.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
from __future__ import annotations
22

3-
import _thread
43
import json
54
import os
65
import shutil
76
import signal
87
import subprocess
98
import sys
109
import tempfile
11-
import threading
1210
import time
1311
from collections.abc import Callable
1412
from itertools import takewhile
@@ -920,16 +918,6 @@ def write_custom_host_project(*, project_dir: Path, model_count: int, rule_body:
920918
)
921919

922920

923-
def interrupt_main_after(delay_seconds: float) -> Callable[[Path], None]:
924-
"""Return a host publish hook that interrupts the main thread after a delay."""
925-
926-
def interrupt(path: Path) -> None:
927-
del path
928-
threading.Timer(delay_seconds, _thread.interrupt_main).start()
929-
930-
return interrupt
931-
932-
933921
def child_processes() -> frozenset[int]:
934922
"""Return the live child processes of this process."""
935923

‎tests/integration/src/sqlbuild/cli/commands/main/test_rules.py‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,17 @@
5353
child_processes,
5454
compile_finding_keys,
5555
compile_findings,
56-
interrupt_main_after,
5756
observe_hosts,
5857
write_custom_host_project,
5958
write_named_selector_project,
6059
write_project_files,
6160
write_project_scratch_rules,
6261
write_python_rules_project,
6362
)
63+
from tests.unit.src.sqlbuild.rule_engine._helpers.host.interrupts.helpers import (
64+
default_interrupt_handling,
65+
interrupt_when,
66+
)
6467

6568
_CACHE_RULES_TOML: str = (
6669
'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n'
@@ -1388,8 +1391,7 @@ def failed_compile() -> tuple[int, dict[str, Any]]:
13881391
description="busy split hosts stop after an interrupt",
13891392
model_count=45,
13901393
hosts=3,
1391-
interrupt_after_seconds=1.5,
1392-
expected_max_seconds=12.0,
1394+
expected_host_launches=3,
13931395
),
13941396
],
13951397
ids=lambda case: case.description,
@@ -1399,26 +1401,34 @@ def test_given_busy_split_hosts_when_compile_is_interrupted_then_hosts_stop_and_
13991401
tmp_path: Path,
14001402
monkeypatch: pytest.MonkeyPatch,
14011403
) -> None:
1404+
project_dir: Path = tmp_path / "orders"
1405+
project_dir.mkdir()
1406+
started: Path = tmp_path / "started"
1407+
started.mkdir()
14021408
write_custom_host_project(
1403-
project_dir=tmp_path,
1409+
project_dir=project_dir,
14041410
model_count=test_case.model_count,
1405-
rule_body="total = 0\nfor index in range(10_000_000):\n total += index\nreturn []",
1411+
rule_body=(
1412+
"from pathlib import Path\n"
1413+
f"(Path({str(started)!r}) / model.name).mkdir()\n"
1414+
"total = 0\nfor index in range(10_000_000):\n total += index\nreturn []"
1415+
),
14061416
)
14071417
children_before: frozenset[int] = child_processes()
1408-
hosts: CustomHostObserver = observe_hosts(
1409-
monkeypatch=monkeypatch,
1410-
cores=test_case.hosts,
1411-
publish_inputs=interrupt_main_after(test_case.interrupt_after_seconds),
1412-
)
1413-
started: float = time.monotonic()
1418+
hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=test_case.hosts)
14141419

1415-
with pytest.raises(KeyboardInterrupt):
1416-
_ = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"])
1420+
with (
1421+
default_interrupt_handling(),
1422+
interrupt_when(lambda: len(tuple(started.iterdir())) >= test_case.hosts) as sent,
1423+
pytest.raises(KeyboardInterrupt),
1424+
):
1425+
_ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"])
14171426

1418-
assert time.monotonic() - started < test_case.expected_max_seconds
1419-
assert len(hosts.launches()) == test_case.hosts
1427+
assert sent.is_set()
1428+
assert len(hosts.launches()) == test_case.expected_host_launches
1429+
assert len(tuple(started.iterdir())) < test_case.model_count
14201430
assert child_processes() <= children_before
1421-
assert not tuple((tmp_path / "target" / "rules-cache" / "host-inputs").iterdir())
1431+
assert not tuple((project_dir / "target" / "rules-cache" / "host-inputs").iterdir())
14221432

14231433

14241434
@pytest.mark.parametrize(

0 commit comments

Comments
 (0)