diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs index ca192c2c4e..e2b6d9e56d 100644 --- a/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs @@ -3,7 +3,7 @@ use std::cell::Cell; use pyo3::exceptions::PyKeyboardInterrupt; -use pyo3::prelude::{PyResult, Python}; +use pyo3::prelude::{Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; thread_local! { pub(crate) static INTERRUPTED: Cell = const { Cell::new(false) }; @@ -23,3 +23,19 @@ pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> { } Ok(()) } + +/// Put `signal`'s previous Python handler back, unless something replaced `installed` meanwhile. Fails when a pending handler raised first, as `signal.signal` runs pending handlers before it changes one. +pub(crate) fn restore_signal_handler( + py: Python<'_>, + signal: i64, + (installed, previous): (&Py, &Py), +) -> PyResult<()> { + let module: pyo3::Bound<'_, PyModule> = py.import("signal")?; + if module + .call_method1("getsignal", (signal,))? + .is(installed.bind(py)) + { + module.call_method1("signal", (signal, previous.bind(py)))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-base/src/native_host/models.rs b/crates/sqlbuild-python-base/src/native_host/models.rs index 32544d9016..8e6757a6fa 100644 --- a/crates/sqlbuild-python-base/src/native_host/models.rs +++ b/crates/sqlbuild-python-base/src/native_host/models.rs @@ -2,11 +2,13 @@ use crate::native_host::_helpers::interrupt::INTERRUPTED; use crate::native_host::_helpers::interrupt::SIGINT; +use crate::native_host::_helpers::interrupt::restore_signal_handler; use pyo3::exceptions::PyKeyboardInterrupt; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::sync::PyOnceLock; @@ -107,22 +109,83 @@ impl InterruptHandler { impl Drop for InterruptHandler { fn drop(&mut self) { Python::attach(|py| { - let restored: PyResult<()> = (|| { - let signal = py.import("signal")?; - if signal - .call_method1("getsignal", (SIGINT,))? - .is(self.installed.bind(py)) - { - signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?; - } - Ok(()) - })(); - drop(restored); + drop(restore_signal_handler( + py, + SIGINT, + (&self.installed, &self.previous), + )); }); INTERRUPTED.set(false); } } +/// A command's handler on an otherwise unused signal that raises the command's error, so another thread can interrupt the main thread while SIGINT is ignored, without changing how SIGINT is handled. +#[derive(Debug)] +pub struct WakeHandler { + pub signal: i64, + pub(crate) previous: Py, + pub(crate) installed: Py, +} + +impl WakeHandler { + /// The signal that interrupts the main thread: SIGINT where Python handles it, else the signal of a wake handler raising `error`, installed here; `None` when neither is available. + pub fn interrupting_signal( + py: Python<'_>, + error: fn() -> PyErr, + ) -> PyResult<(Option, Option)> { + let signal = py.import("signal")?; + if signal.call_method1("getsignal", (SIGINT,))?.is_callable() { + return Ok((Some(SIGINT), None)); + } + let wake: Option = Self::install(py, error)?; + Ok((wake.as_ref().map(|wake| wake.signal), wake)) + } + + /// Install the handler on SIGRTMIN, or SIGUSR2 where there is none, while that signal takes its default action. + fn install(py: Python<'_>, error: fn() -> PyErr) -> PyResult> { + let signal = py.import("signal")?; + let name: &str = if signal.hasattr("SIGRTMIN")? { + "SIGRTMIN" + } else if signal.hasattr("SIGUSR2")? { + "SIGUSR2" + } else { + return Ok(None); + }; + let number: i64 = signal.getattr(name)?.extract()?; + let previous = signal.call_method1("getsignal", (number,))?; + if !previous.is(signal.getattr("SIG_DFL")?) { + return Ok(None); + } + let installed = PyCFunction::new_closure( + py, + None, + None, + move |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + Err(error()) + }, + )?; + signal.call_method1("signal", (number, &installed))?; + Ok(Some(Self { + signal: number, + previous: previous.unbind(), + installed: installed.into_any().unbind(), + })) + } +} + +impl Drop for WakeHandler { + /// Restore the previous handler. A wake still pending raises on the first attempt and is consumed, so the second attempt restores. + fn drop(&mut self) { + Python::attach(|py| { + let handlers = (&self.installed, &self.previous); + drop( + restore_signal_handler(py, self.signal, handlers) + .or_else(|_pending| restore_signal_handler(py, self.signal, handlers)), + ); + }); + } +} + impl std::fmt::Debug for AuditProjection { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs index 512886cef8..ab7d77a9f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs @@ -16,6 +16,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::WakeHandler; use sqlbuild_python_base::records::main::fields::field::field as native_field; create_exception!( @@ -69,11 +70,12 @@ fn registered_flags() -> Option<&'static SigtermFlags> { .as_ref() } -/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. Holds only Rust values: while it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again. +/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. While it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again. pub(crate) struct TerminationHandler { flags: Option<&'static SigtermFlags>, stop: Arc, watcher: Option>, + wake: Option, } impl TerminationHandler { @@ -86,6 +88,7 @@ impl TerminationHandler { if let Some(watcher) = self.watcher.take() { py.detach(|| drop(watcher.join())); } + drop(self.wake.take()); terminated() } } @@ -96,6 +99,7 @@ impl Drop for TerminationHandler { if let Some(watcher) = self.watcher.take() { Python::attach(|py| py.detach(|| drop(watcher.join()))); } + drop(self.wake.take()); if let Some(flags) = self.flags { flags.default.store(true, Ordering::SeqCst); flags.terminated.store(false, Ordering::SeqCst); @@ -103,11 +107,11 @@ impl Drop for TerminationHandler { } } -/// Where the watcher sends the interrupt: the main thread, and whether Python handles Ctrl+C there. +/// The signal the watcher sends the main thread: SIGINT where Python handles Ctrl+C, else the wake signal. #[derive(Clone, Copy)] struct MainThread { ident: u64, - handles_interrupt: bool, + signal: Option, } /// Once SIGTERM arrives: let a second one end the process at once, cancel every running statement, and interrupt the main thread the way Ctrl+C does, where the platform can signal one thread. @@ -119,9 +123,9 @@ fn watch(flags: &SigtermFlags, stop: &AtomicBool, main: MainThread) { let cancelled = host_entry(py, ExecutorHost::CancelRunningStatements) .and_then(|cancel| cancel.call0()); drop(cancelled); - if main.handles_interrupt { + if let Some(signal) = main.signal { let interrupted = imported(py, "signal", "pthread_kill") - .and_then(|kill| kill.call1((main.ident, SIGINT_NUMBER))); + .and_then(|kill| kill.call1((main.ident, signal))); drop(interrupted); } }); @@ -148,11 +152,14 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult { flags: None, stop, watcher: None, + wake: None, }); }; + let (signal, wake): (Option, Option) = + WakeHandler::interrupting_signal(py, || CommandTerminated::new_err(TERMINATED_MESSAGE))?; let main = MainThread { ident: native_field(&main_thread, "ident")?.extract()?, - handles_interrupt: action(SIGINT_NUMBER)?.is_callable(), + signal, }; flags.terminated.store(false, Ordering::SeqCst); let watched: Arc = Arc::clone(&stop); @@ -164,6 +171,7 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult { flags: Some(flags), stop, watcher: Some(watcher), + wake, }) } diff --git a/fensu.toml b/fensu.toml index b5f37e48fd..c51c3c0203 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1655,6 +1655,11 @@ rule = "FFT104" path = "tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py" reason = "The disposable HTTPS gateway and OIDC provider dispatch real HTTP protocol messages and authentication transports; assertions in the parametrized installed journey remain branch-free." +[[targets.python.rule_exceptions]] +rule = "FFT104" +path = "tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py" +reason = "The interrupter thread polls for the event that starts the interrupt and must not interrupt once its block has ended; the tests using it remain branch-free." + [targets.rust] analyzer = "rust" roots = ["crates"] @@ -1670,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 66756 } +thresholds = { max_crate_code_lines = 66764 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index bfeaf3eb62..dd4179906a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -155,9 +155,11 @@ can contain sensitive SQL, output and audit evidence. A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it cancels the statements it is running, records its real end time, its output and exit code 143, -prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process -at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run -stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +prints `error: terminated by SIGTERM`, and exits with code 143. It does this also when it was +started with SIGINT ignored, as a script starts a background job and some process managers start a +service; SIGINT and Ctrl-C then stay ignored. A second SIGTERM ends the process at once. Recall +keeps heartbeating while the command discovers and compiles the project, so a run stopped in any +phase is never mistaken for one that died. An `abandoned` run cannot record its end: its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. ## Selection diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index e2191ebab7..79e06bd725 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -215,8 +215,7 @@ class CustomHostInterruptIntegrationTestCase: description: str model_count: int hosts: int - interrupt_after_seconds: float - expected_max_seconds: float + expected_host_launches: int @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 44970058e9..d9a0ab9067 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1,6 +1,5 @@ from __future__ import annotations -import _thread import json import os import shutil @@ -8,7 +7,6 @@ import subprocess import sys import tempfile -import threading import time from collections.abc import Callable from itertools import takewhile @@ -920,16 +918,6 @@ def write_custom_host_project(*, project_dir: Path, model_count: int, rule_body: ) -def interrupt_main_after(delay_seconds: float) -> Callable[[Path], None]: - """Return a host publish hook that interrupts the main thread after a delay.""" - - def interrupt(path: Path) -> None: - del path - threading.Timer(delay_seconds, _thread.interrupt_main).start() - - return interrupt - - def child_processes() -> frozenset[int]: """Return the live child processes of this process.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index fce0de2a8f..9662e05e3a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -53,7 +53,6 @@ child_processes, compile_finding_keys, compile_findings, - interrupt_main_after, observe_hosts, write_custom_host_project, write_named_selector_project, @@ -61,6 +60,10 @@ write_project_scratch_rules, write_python_rules_project, ) +from tests.unit.src.sqlbuild.rule_engine._helpers.host.interrupts.helpers import ( + default_interrupt_handling, + interrupt_when, +) _CACHE_RULES_TOML: str = ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n' @@ -1388,8 +1391,7 @@ def failed_compile() -> tuple[int, dict[str, Any]]: description="busy split hosts stop after an interrupt", model_count=45, hosts=3, - interrupt_after_seconds=1.5, - expected_max_seconds=12.0, + expected_host_launches=3, ), ], ids=lambda case: case.description, @@ -1399,26 +1401,34 @@ def test_given_busy_split_hosts_when_compile_is_interrupted_then_hosts_stop_and_ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + project_dir: Path = tmp_path / "orders" + project_dir.mkdir() + started: Path = tmp_path / "started" + started.mkdir() write_custom_host_project( - project_dir=tmp_path, + project_dir=project_dir, model_count=test_case.model_count, - rule_body="total = 0\nfor index in range(10_000_000):\n total += index\nreturn []", + rule_body=( + "from pathlib import Path\n" + f"(Path({str(started)!r}) / model.name).mkdir()\n" + "total = 0\nfor index in range(10_000_000):\n total += index\nreturn []" + ), ) children_before: frozenset[int] = child_processes() - hosts: CustomHostObserver = observe_hosts( - monkeypatch=monkeypatch, - cores=test_case.hosts, - publish_inputs=interrupt_main_after(test_case.interrupt_after_seconds), - ) - started: float = time.monotonic() + hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=test_case.hosts) - with pytest.raises(KeyboardInterrupt): - _ = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + with ( + default_interrupt_handling(), + interrupt_when(lambda: len(tuple(started.iterdir())) >= test_case.hosts) as sent, + pytest.raises(KeyboardInterrupt), + ): + _ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) - assert time.monotonic() - started < test_case.expected_max_seconds - assert len(hosts.launches()) == test_case.hosts + assert sent.is_set() + assert len(hosts.launches()) == test_case.expected_host_launches + assert len(tuple(started.iterdir())) < test_case.model_count assert child_processes() <= children_before - assert not tuple((tmp_path / "target" / "rules-cache" / "host-inputs").iterdir()) + assert not tuple((project_dir / "target" / "rules-cache" / "host-inputs").iterdir()) @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index e161327af8..c07441efee 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -1,7 +1,10 @@ +import signal from collections.abc import Callable from dataclasses import dataclass from pathlib import Path +from tests.integration.src.sqlbuild.cli.recall.helpers import PYTHON_CLI + @dataclass(frozen=True) class RecallOutputTestCase: @@ -198,6 +201,14 @@ class SlowReplacementTestCase: expected_max_seconds: float +@dataclass(frozen=True) +class InterruptedSinkTestCase: + description: str + shutdown_seconds: float + sink_sleep_seconds: str + expected_exit_code: int + + @dataclass(frozen=True) class ThirdPartyLogTestCase: description: str @@ -369,6 +380,9 @@ class TerminatedCommandTestCase: environment: dict[str, str] wait_sql: str expected_output: bytes + unfinished_output: bytes + interrupts: signal.Handlers = signal.SIG_DFL + program: Path = PYTHON_CLI @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 569ed78183..d88e972708 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -26,6 +26,8 @@ MEBIBYTE: int = 1024 * 1024 HANG_SECONDS: int = 600 HANG_DUMP_GRACE_SECONDS: int = 30 +# Only a hang takes this long: a signalled build stops within a second or two. +STOP_HANG_SECONDS: int = 120 EMPTY_RESULT: bytes = b'{"rows": []}' NUMBERED_LINE: re.Pattern[bytes] = re.compile(rb"^(\d+) std(?:out|err)$", re.MULTILINE) PROJECT_TOML: str = ( @@ -61,6 +63,7 @@ f" sys.stdout.write({EMITTED_TAIL.decode()!r})\n" " sys.stdout.flush()\n" ' time.sleep(float(os.environ.get("ORDERS_TASK_SLEEP_SECONDS", "0")))\n' + ' sys.stdout.write(os.environ.get("ORDERS_TASK_AFTER_SLEEP", ""))\n' ' _write_concurrently(int(os.environ.get("ORDERS_TASK_THREAD_LINES", "0")))\n' ' sys.stdout.write("y" * int(os.environ.get("ORDERS_TASK_STDOUT_BYTES", "0")))\n' " sys.stdout.flush()\n" @@ -118,7 +121,9 @@ def environment(**overrides: str) -> dict[str, str]: def run(project_dir: Path, *args: str, **env: str) -> subprocess.CompletedProcess[bytes]: """Run the Python CLI once in the project and capture exact streams, or fail with its stacks.""" - process: subprocess.Popen[bytes] = start(project_dir, *args, **env) + process: subprocess.Popen[bytes] = start( + project_dir, *args, interrupts=signal.SIG_DFL, program=PYTHON_CLI, **env + ) try: stdout, stderr = process.communicate(timeout=HANG_SECONDS) except subprocess.TimeoutExpired: @@ -142,16 +147,25 @@ def hang_dump(process: subprocess.Popen[bytes]) -> str: return stderr.decode(errors="replace") -def start(project_dir: Path, *args: str, **env: str) -> subprocess.Popen[bytes]: - """Start the Python CLI in its own process group.""" +def start( + project_dir: Path, + *args: str, + interrupts: signal.Handlers = signal.SIG_DFL, + program: Path = PYTHON_CLI, + **env: str, +) -> subprocess.Popen[bytes]: + """Start `program` in its own process group with SIGINT set to `interrupts`. + + The default gives the CLI Ctrl-C handling even when this runner ignores SIGINT.""" return subprocess.Popen( - [str(PYTHON_CLI), *args], + [str(program), *args], cwd=project_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment(**env), start_new_session=True, + preexec_fn=lambda: signal.signal(signal.SIGINT, interrupts), ) @@ -251,36 +265,64 @@ def write_slow_import_sink(project_dir: Path) -> Path: sink: Path = project_dir / "sinks" / "slow_import.py" sink.parent.mkdir(parents=True, exist_ok=True) _ = sink.write_text( - "import os\nimport time\n\n" - "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n", + "import os\nimport sys\nimport time\n\n" + "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n" + "sys.stdout.write(os.environ.get('ORDERS_IMPORT_AFTER_SLEEP', ''))\n", encoding="utf-8", ) return project_dir def terminated_build( - project_dir: Path, *, wait_sql: str, **env: str -) -> tuple[subprocess.CompletedProcess[bytes], float]: - """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row, and the - seconds it took to stop.""" + project_dir: Path, + *, + wait_sql: str, + interrupts: signal.Handlers = signal.SIG_DFL, + program: Path = PYTHON_CLI, + **env: str, +) -> subprocess.CompletedProcess[bytes]: + """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row.""" - process: subprocess.Popen[bytes] = start(project_dir, "build", **env) + process: subprocess.Popen[bytes] = start( + project_dir, "build", interrupts=interrupts, program=program, **env + ) try: _ = wait_for(wait_sql, project_dir, timeout=120) process.send_signal(signal.SIGTERM) - sent: float = time.monotonic() - stdout, stderr = process.communicate(timeout=60) + stdout, stderr = process.communicate(timeout=STOP_HANG_SECONDS) except subprocess.TimeoutExpired: - raise AssertionError(f"a terminated build kept running:\n{hang_dump(process)}") from None - completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( - process.args, process.returncode, stdout, stderr + raise AssertionError(f"a terminated build hung:\n{hang_dump(process)}") from None + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + + +def main_thread_state(pid: int) -> tuple[str, str, str]: + """The scheduler state and user and system CPU ticks of the process's main thread.""" + + fields: list[str] = ( + Path(f"/proc/{pid}/task/{pid}/stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split() ) - return completed, time.monotonic() - sent + return fields[0], fields[11], fields[12] + +def wait_until_blocked(pid: int, *, samples: int = 5) -> None: + """Wait until the main thread has slept without using CPU for `samples` polls in a row. -def terminated_while_closing(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float]: - """A build sent SIGTERM while it waits for a locked Recall store at exit, and the seconds it - took to stop.""" + A thread waiting for the CPU reports `R`, so load does not pass for blocked.""" + + seen: list[tuple[str, str, str]] = [main_thread_state(pid)] + deadline: float = time.monotonic() + STOP_HANG_SECONDS + while (seen[-samples:] != [seen[-1]] * samples or seen[-1][0] != "S") and ( + time.monotonic() < deadline + ): + time.sleep(0.1) + seen.append(main_thread_state(pid)) + assert seen[-samples:] == [seen[-1]] * samples and seen[-1][0] == "S", ( + f"the build's main thread never blocked, so it hung: {seen[-samples:]}" + ) + + +def terminated_while_closing(project_dir: Path) -> subprocess.CompletedProcess[bytes]: + """A build sent SIGTERM while it waits for a locked Recall store at exit.""" with held_store_lock(project_dir): process: subprocess.Popen[bytes] = start(project_dir, "build", PYTHONUNBUFFERED="1") @@ -288,36 +330,37 @@ def terminated_while_closing(project_dir: Path) -> tuple[subprocess.CompletedPro printed: bytes = b"" while b"Completed" not in printed and process.poll() is None: printed += process.stdout.readline() - time.sleep(1) + wait_until_blocked(process.pid) process.send_signal(signal.SIGTERM) - sent: float = time.monotonic() try: - stdout, stderr = process.communicate(timeout=60) + stdout, stderr = process.communicate(timeout=STOP_HANG_SECONDS) except subprocess.TimeoutExpired: - raise AssertionError(f"a closing build kept running:\n{hang_dump(process)}") from None - completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( - process.args, process.returncode, printed + stdout, stderr - ) - return completed, time.monotonic() - sent + raise AssertionError(f"a closing build hung:\n{hang_dump(process)}") from None + return subprocess.CompletedProcess(process.args, process.returncode, printed + stdout, stderr) -def interrupted_build(project_dir: Path, **env: str) -> tuple[int | None, float]: - """Exit code of a build sent SIGINT after its first logged event, and seconds to stop.""" +def interrupted_build(project_dir: Path, **env: str) -> int | None: + """Exit code of a build sent SIGINT once its task has printed; `None` when it hung.""" - log: Path = project_dir / "events.log" - process: subprocess.Popen[bytes] = start(project_dir, "build", **env) + process: subprocess.Popen[bytes] = start( + project_dir, + "build", + interrupts=signal.SIG_DFL, + program=PYTHON_CLI, + PYTHONUNBUFFERED="1", + **env, + ) try: - deadline: float = time.monotonic() + 120 - while not log.exists() and time.monotonic() < deadline: - time.sleep(0.1) - time.sleep(1) - interrupted: float = time.monotonic() + assert process.stdout is not None + printed: bytes = b"" + while EMITTED_TAIL not in printed and process.poll() is None: + printed += process.stdout.readline() os.killpg(process.pid, signal.SIGINT) try: - _ = process.communicate(timeout=30) + _ = process.communicate(timeout=STOP_HANG_SECONDS) except subprocess.TimeoutExpired: - return None, time.monotonic() - interrupted - return process.returncode, time.monotonic() - interrupted + return None + return process.returncode finally: with contextlib.suppress(ProcessLookupError): os.killpg(process.pid, signal.SIGKILL) diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py index f993680526..1d0df8d2f3 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py @@ -1,5 +1,6 @@ from __future__ import annotations +import signal import subprocess import time from pathlib import Path @@ -9,6 +10,7 @@ from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.integration.src.sqlbuild.cli.recall._test_types import ( FailingReplacementTestCase, + InterruptedSinkTestCase, ReplacementLaunchRecordTestCase, ReplacementLifecycleTestCase, ReplacementOutputTestCase, @@ -224,17 +226,17 @@ def test_given_a_lifecycle_sink_slower_than_the_budget_when_building_then_nothin @pytest.mark.parametrize( "test_case", [ - SlowReplacementTestCase( + InterruptedSinkTestCase( description="Ctrl-C stops the build while the sink sleeps", shutdown_seconds=1.0, sink_sleep_seconds="30", - expected_max_seconds=10.0, + expected_exit_code=-signal.SIGINT, ), ], ids=lambda case: case.description, ) def test_given_a_slow_lifecycle_sink_when_interrupted_then_the_build_stops( - test_case: SlowReplacementTestCase, tmp_path: Path + test_case: InterruptedSinkTestCase, tmp_path: Path ) -> None: budget: str = f'\n[recall]\nshutdown_timeout = "{int(test_case.shutdown_seconds * 1000)}ms"\n' project_dir: Path = write_sinks( @@ -242,14 +244,14 @@ def test_given_a_slow_lifecycle_sink_when_interrupted_then_the_build_stops( {"recall_events": SLOW_LIFECYCLE_SINK, "event_log": EVENT_LOG_SINK}, ) - exit_code, stopped_after = interrupted_build( + exit_code: int | None = interrupted_build( project_dir, ORDERS_SINK_SLEEP_SECONDS=test_case.sink_sleep_seconds, ORDERS_TASK_SLEEP_SECONDS="60", ) - assert exit_code not in (None, 0) - assert stopped_after < test_case.expected_max_seconds + assert exit_code is not None, "the interrupted build hung" + assert exit_code == test_case.expected_exit_code if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py index 42ca4741b9..0cfcd34df3 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -1,4 +1,5 @@ import json +import signal import subprocess from pathlib import Path @@ -15,6 +16,7 @@ ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( EMITTED_TAIL, + NATIVE_SQB, ORDERS_MODEL, recall_rows, run, @@ -90,21 +92,40 @@ def test_given_selectors_when_building_then_the_effective_selection_is_recorded( TerminatedCommandTestCase( description="during project discovery", orders_model=ORDERS_MODEL, - environment={"ORDERS_IMPORT_SECONDS": "60"}, + environment={ + "ORDERS_IMPORT_SECONDS": "60", + "ORDERS_IMPORT_AFTER_SLEEP": "slow sink imported\n", + }, wait_sql=( "SELECT invocation_id FROM invocations WHERE status = 'running' " "AND (julianday(last_heartbeat_at) - julianday(started_at)) * 86400 >= 2" ), expected_output=b"", + unfinished_output=b"slow sink imported", ), TerminatedCommandTestCase( description="while a task runs", orders_model=ORDERS_MODEL, - environment={"ORDERS_TASK_SLEEP_SECONDS": "60"}, - wait_sql=( - "SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'" - ), + environment={ + "ORDERS_TASK_SLEEP_SECONDS": "60", + "ORDERS_TASK_AFTER_SLEEP": "slept through\n", + }, + wait_sql="SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'", expected_output=EMITTED_TAIL, + unfinished_output=b"slept through", + ), + TerminatedCommandTestCase( + description="while a task runs, sqb started with SIGINT ignored", + orders_model=ORDERS_MODEL, + environment={ + "ORDERS_TASK_SLEEP_SECONDS": "60", + "ORDERS_TASK_AFTER_SLEEP": "slept through\n", + }, + wait_sql="SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'", + expected_output=EMITTED_TAIL, + unfinished_output=b"slept through", + interrupts=signal.SIG_IGN, + program=NATIVE_SQB, ), TerminatedCommandTestCase( description="while a long warehouse statement runs", @@ -118,6 +139,7 @@ def test_given_selectors_when_building_then_the_effective_selection_is_recorded( "WHERE statement_kind = 'CREATE' AND ended_at IS NULL" ), expected_output=b"", + unfinished_output=b"Completed", ), ], ids=lambda case: case.description, @@ -128,9 +150,11 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit project_dir: Path = write_slow_import_sink(write_project(tmp_path / "orders_shop")) _ = (project_dir / "models" / "orders.sql").write_text(test_case.orders_model, encoding="utf-8") - terminated, stop_seconds = terminated_build( + terminated: subprocess.CompletedProcess[bytes] = terminated_build( project_dir, wait_sql=test_case.wait_sql, + interrupts=test_case.interrupts, + program=test_case.program, SQLBUILD_RECALL_HEARTBEAT_SECONDS="0.5", **test_case.environment, ) @@ -143,10 +167,14 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit starts: list[list[object]] = recall_rows( project_dir, "SELECT COUNT(*) FROM recall_events WHERE event_type = 'invocation_started'" ) + running_statements: list[list[object]] = recall_rows( + project_dir, "SELECT COUNT(*) FROM statements WHERE ended_at IS NULL" + ) output: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "output", "last") assert terminated.returncode == 143 - assert stop_seconds < 10 + assert test_case.unfinished_output not in terminated.stdout + output.stdout + assert running_statements == [[0]] assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") assert b"Traceback" not in terminated.stderr assert recorded == [["cancelled", 15, 143, 0, 1, 1]] @@ -160,7 +188,7 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit [ ClosingTerminationTestCase( description="a build waiting for a locked store at exit", - shutdown_timeout="60s", + shutdown_timeout="600s", expected_exit_code=143, ), ], @@ -174,13 +202,12 @@ def test_given_sigterm_while_recall_closes_when_a_build_has_finished_then_it_exi extra_toml=f'\n[recall]\nshutdown_timeout = "{test_case.shutdown_timeout}"\n', ) - terminated, stop_seconds = terminated_while_closing(project_dir) + terminated: subprocess.CompletedProcess[bytes] = terminated_while_closing(project_dir) assert b"Completed" in terminated.stdout assert terminated.returncode == test_case.expected_exit_code, terminated.stderr.decode( errors="replace" ) - assert stop_seconds < 10 assert b"Traceback" not in terminated.stderr assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py index c0d9afb685..c18bbe10dc 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py @@ -40,14 +40,15 @@ class HostCancellationTestCase: @dataclass(frozen=True) class HostInterruptTestCase: - """An interrupt while split hosts are still busy with slow invocations.""" + """An interrupt sent once `interrupt_at_hosts` host processes are busy with slow invocations.""" description: str model_count: int hosts: int delay_seconds: float - interrupt_after_seconds: float - expected_max_seconds: float + interrupt_at_hosts: int + expected_host_launches: int + failing: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index 8fe68caed4..c6bf7c5a0d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -1,10 +1,8 @@ """Builders for custom-rule host partitioning tests.""" -import _thread import os import random import shutil -import threading import time from collections.abc import Callable from operator import attrgetter @@ -38,6 +36,8 @@ @rule(code="{_MODEL_RULE}", message="order models need an even suffix", remediation="Rename it.") def even_suffix(*, model: Model, ctx: RuleContext) -> list[Finding]: + if STARTED_DIR: + os.mkdir(Path(STARTED_DIR) / f"{{model.name}}.{{os.getpid()}}") if model.name in FAILING: raise ValueError(f"cannot inspect {{model.name}}") time.sleep(DELAY_SECONDS) @@ -68,13 +68,19 @@ def write_order_rules( records_module_state: bool = False, detects_duplicates: bool = False, delay_seconds: float = 0.0, + started_dir: Path | str = "", ) -> None: - """Write one model rule and one project rule with optional failures, state, or delays.""" + """Write one model rule and one project rule with optional failures, state, or delays. + + With `started_dir`, each model invocation first creates the folder `.` there; + the host's hermeticity guard refuses opening files but allows creating folders.""" rules: Path = root / "rules" / "orders.py" rules.parent.mkdir(parents=True, exist_ok=True) rules.write_text( - f"import time\n\nFAILING = {set(failing)!r}\nRECORD_SEEN = {records_module_state!r}\n" + "import os\nimport time\nfrom pathlib import Path\n\n" + f"STARTED_DIR = {str(started_dir)!r}\n" + f"FAILING = {set(failing)!r}\nRECORD_SEEN = {records_module_state!r}\n" f"DETECT_DUPLICATES = {detects_duplicates!r}\nDELAY_SECONDS = {delay_seconds!r}\n" f"SEEN: dict[str, bool] = {{}}\n{_RULES_MODULE}", encoding="utf-8", @@ -188,14 +194,10 @@ def fail(path: Path) -> None: return fail, failures -def interrupting_publish(delay_seconds: float) -> Callable[[Path], None]: - """Return a hook that interrupts the main thread once hosts have run for a while.""" - - def interrupt(path: Path) -> None: - del path - threading.Timer(delay_seconds, _thread.interrupt_main).start() +def started_hosts(started_dir: Path) -> int: + """Return how many host processes started an invocation, from `.` folders.""" - return interrupt + return len({path.suffix for path in started_dir.iterdir()}) def record_signals(*, monkeypatch: pytest.MonkeyPatch) -> list[tuple[int, int]]: diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py new file mode 100644 index 0000000000..3374193072 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py @@ -0,0 +1,50 @@ +"""Interrupt the main thread once an observed event happens, the way Ctrl-C does.""" + +import _thread +import contextlib +import signal +import threading +from collections.abc import Callable, Iterator + + +@contextlib.contextmanager +def default_interrupt_handling() -> Iterator[None]: + """Give the main thread Python's default Ctrl-C handling while the block runs. + + Test runners often start with SIGINT ignored; `_thread.interrupt_main` then does nothing.""" + + previous: object = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + yield + finally: + _ = signal.signal(signal.SIGINT, previous) + + +@contextlib.contextmanager +def interrupt_when(ready: Callable[[], bool]) -> Iterator[threading.Event]: + """Interrupt the main thread once `ready()` holds. + + The event is set when the interrupt was sent; leaving the block stops the watcher, so no + interrupt arrives after it.""" + + sent: threading.Event = threading.Event() + stop: threading.Event = threading.Event() + lock: threading.Lock = threading.Lock() + + def watch() -> None: + while not stop.wait(0.02): + if ready(): + with lock: + if not stop.is_set(): + sent.set() + _thread.interrupt_main() + return + + watcher: threading.Thread = threading.Thread(target=watch, daemon=True) + watcher.start() + try: + yield sent + finally: + with lock: + stop.set() + watcher.join() diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 5ba1f73053..6569d66040 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -30,13 +30,17 @@ failed_publish, failed_publish_without_directory, host_inputs_left, - interrupting_publish, orders_project, record_signals, + started_hosts, touch_files, use_hosts, write_order_rules, ) +from tests.unit.src.sqlbuild.rule_engine._helpers.host.interrupts.helpers import ( + default_interrupt_handling, + interrupt_when, +) @pytest.mark.parametrize( @@ -233,8 +237,8 @@ def test_given_failing_host_when_others_are_busy_then_they_stop_before_later_inv model_count=60, hosts=3, delay_seconds=1.0, - interrupt_after_seconds=1.5, - expected_max_seconds=10.0, + interrupt_at_hosts=3, + expected_host_launches=3, ), ], ids=lambda case: case.description, @@ -242,25 +246,76 @@ def test_given_failing_host_when_others_are_busy_then_they_stop_before_later_inv def test_given_busy_split_hosts_when_interrupted_then_hosts_stop_and_no_process_remains( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostInterruptTestCase ) -> None: - write_order_rules(root=tmp_path, delay_seconds=test_case.delay_seconds) + started: Path = tmp_path / "started" + started.mkdir() + write_order_rules(root=tmp_path, delay_seconds=test_case.delay_seconds, started_dir=started) project: CompiledProjectView = orders_project(model_count=test_case.model_count) children_before: frozenset[int] = child_processes() - hosts: CustomHostObserver = use_hosts( - monkeypatch=monkeypatch, - hosts=test_case.hosts, - publish_inputs=interrupting_publish(test_case.interrupt_after_seconds), - ) - started: float = time.monotonic() + hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) - with pytest.raises(KeyboardInterrupt): + with ( + default_interrupt_handling(), + interrupt_when(lambda: started_hosts(started) >= test_case.interrupt_at_hosts) as sent, + pytest.raises(KeyboardInterrupt), + ): _ = evaluate_order_rules(project=project, root=tmp_path) - assert time.monotonic() - started < test_case.expected_max_seconds - assert len(hosts.launches()) == test_case.hosts + assert sent.is_set() + assert len(hosts.launches()) == test_case.expected_host_launches + assert len(tuple(started.iterdir())) < test_case.model_count assert child_processes() <= children_before assert not host_inputs_left(tmp_path) +@pytest.mark.parametrize( + "test_case", + [ + HostInterruptTestCase( + description="interrupt while the single-host fallback runs", + model_count=30, + hosts=3, + delay_seconds=0.2, + interrupt_at_hosts=4, + expected_host_launches=4, + failing=("orders_029",), + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupt_while_hosts_are_busy_when_the_run_ends_then_the_interrupt_is_raised( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostInterruptTestCase +) -> None: + started: Path = tmp_path / "started" + started.mkdir() + write_order_rules( + root=tmp_path, + delay_seconds=test_case.delay_seconds, + started_dir=started, + failing=test_case.failing, + ) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) + hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) + outcomes: list[str] = [] + + with default_interrupt_handling(): + try: + with interrupt_when( + lambda: started_hosts(started) >= test_case.interrupt_at_hosts + ) as sent: + try: + _ = evaluate_order_rules(project=project, root=tmp_path, cache_enabled=False) + except RulesError: + outcomes.append("RulesError") + for _ in range(100): + time.sleep(0.01) # a pending interrupt is raised between bytecodes + except KeyboardInterrupt: + outcomes.append("KeyboardInterrupt") + + assert sent.is_set() + assert len(hosts.launches()) == test_case.expected_host_launches + assert outcomes[-1:] == ["KeyboardInterrupt"] + + @pytest.mark.parametrize( "test_case", ( diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index fcfcbee313..528720c34b 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -134,9 +134,11 @@ can contain sensitive SQL, output and audit evidence. A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it cancels the statements it is running, records its real end time, its output and exit code 143, -prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process -at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run -stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +prints `error: terminated by SIGTERM`, and exits with code 143. It does this also when it was +started with SIGINT ignored, as a script starts a background job and some process managers start a +service; SIGINT and Ctrl-C then stay ignored. A second SIGTERM ends the process at once. Recall +keeps heartbeating while the command discovers and compiles the project, so a run stopped in any +phase is never mistaken for one that died. An `abandoned` run cannot record its end: its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. ## Selection