Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use std::cell::Cell;

use pyo3::exceptions::PyKeyboardInterrupt;
use pyo3::prelude::{PyResult, Python};
use pyo3::prelude::{Py, PyAny, PyAnyMethods, PyModule, PyResult, Python};

thread_local! {
pub(crate) static INTERRUPTED: Cell<bool> = const { Cell::new(false) };
Expand All @@ -23,3 +23,19 @@ pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> {
}
Ok(())
}

/// Put `signal`'s previous Python handler back, unless something replaced `installed` meanwhile. Fails when a pending handler raised first, as `signal.signal` runs pending handlers before it changes one.
pub(crate) fn restore_signal_handler(
py: Python<'_>,
signal: i64,
(installed, previous): (&Py<PyAny>, &Py<PyAny>),
) -> PyResult<()> {
let module: pyo3::Bound<'_, PyModule> = py.import("signal")?;
if module
.call_method1("getsignal", (signal,))?
.is(installed.bind(py))
{
module.call_method1("signal", (signal, previous.bind(py)))?;
}
Ok(())
}
85 changes: 74 additions & 11 deletions crates/sqlbuild-python-base/src/native_host/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@
use crate::native_host::_helpers::interrupt::INTERRUPTED;
use crate::native_host::_helpers::interrupt::SIGINT;
use crate::native_host::_helpers::interrupt::restore_signal_handler;
use pyo3::exceptions::PyKeyboardInterrupt;
use pyo3::prelude::Bound;
use pyo3::prelude::Py;
use pyo3::prelude::PyAny;
use pyo3::prelude::PyAnyMethods;
use pyo3::prelude::PyErr;
use pyo3::prelude::PyResult;
use pyo3::prelude::Python;
use pyo3::sync::PyOnceLock;
Expand Down Expand Up @@ -107,22 +109,83 @@ impl InterruptHandler {
impl Drop for InterruptHandler {
fn drop(&mut self) {
Python::attach(|py| {
let restored: PyResult<()> = (|| {
let signal = py.import("signal")?;
if signal
.call_method1("getsignal", (SIGINT,))?
.is(self.installed.bind(py))
{
signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?;
}
Ok(())
})();
drop(restored);
drop(restore_signal_handler(
py,
SIGINT,
(&self.installed, &self.previous),
));
});
INTERRUPTED.set(false);
}
}

/// A command's handler on an otherwise unused signal that raises the command's error, so another thread can interrupt the main thread while SIGINT is ignored, without changing how SIGINT is handled.
#[derive(Debug)]
pub struct WakeHandler {
pub signal: i64,
pub(crate) previous: Py<PyAny>,
pub(crate) installed: Py<PyAny>,
}

impl WakeHandler {
/// The signal that interrupts the main thread: SIGINT where Python handles it, else the signal of a wake handler raising `error`, installed here; `None` when neither is available.
pub fn interrupting_signal(
py: Python<'_>,
error: fn() -> PyErr,
) -> PyResult<(Option<i64>, Option<Self>)> {
let signal = py.import("signal")?;
if signal.call_method1("getsignal", (SIGINT,))?.is_callable() {
return Ok((Some(SIGINT), None));
}
let wake: Option<Self> = Self::install(py, error)?;
Ok((wake.as_ref().map(|wake| wake.signal), wake))
}

/// Install the handler on SIGRTMIN, or SIGUSR2 where there is none, while that signal takes its default action.
fn install(py: Python<'_>, error: fn() -> PyErr) -> PyResult<Option<Self>> {
let signal = py.import("signal")?;
let name: &str = if signal.hasattr("SIGRTMIN")? {
"SIGRTMIN"
} else if signal.hasattr("SIGUSR2")? {
"SIGUSR2"
} else {
return Ok(None);
};
let number: i64 = signal.getattr(name)?.extract()?;
let previous = signal.call_method1("getsignal", (number,))?;
if !previous.is(signal.getattr("SIG_DFL")?) {
return Ok(None);
}
let installed = PyCFunction::new_closure(
py,
None,
None,
move |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> {
Err(error())
},
)?;
signal.call_method1("signal", (number, &installed))?;
Ok(Some(Self {
signal: number,
previous: previous.unbind(),
installed: installed.into_any().unbind(),
}))
}
}

impl Drop for WakeHandler {
/// Restore the previous handler. A wake still pending raises on the first attempt and is consumed, so the second attempt restores.
fn drop(&mut self) {
Python::attach(|py| {
let handlers = (&self.installed, &self.previous);
drop(
restore_signal_handler(py, self.signal, handlers)
.or_else(|_pending| restore_signal_handler(py, self.signal, handlers)),
);
});
}
}

impl std::fmt::Debug for AuditProjection {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
Expand Down
20 changes: 14 additions & 6 deletions crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use sqlbuild_host::entries::models::ExecutorHost;
use crate::bindings::_helpers::recall::console::write_stderr;
use sqlbuild_python_base::boundary::main::imported::imported;
use sqlbuild_python_base::native_host::main::host_entry::host_entry;
use sqlbuild_python_base::native_host::models::WakeHandler;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

create_exception!(
Expand Down Expand Up @@ -69,11 +70,12 @@ fn registered_flags() -> Option<&'static SigtermFlags> {
.as_ref()
}

/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. Holds only Rust values: while it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again.
/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. While it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again.
pub(crate) struct TerminationHandler {
flags: Option<&'static SigtermFlags>,
stop: Arc<AtomicBool>,
watcher: Option<JoinHandle<()>>,
wake: Option<WakeHandler>,
}

impl TerminationHandler {
Expand All @@ -86,6 +88,7 @@ impl TerminationHandler {
if let Some(watcher) = self.watcher.take() {
py.detach(|| drop(watcher.join()));
}
drop(self.wake.take());
terminated()
}
}
Expand All @@ -96,18 +99,19 @@ impl Drop for TerminationHandler {
if let Some(watcher) = self.watcher.take() {
Python::attach(|py| py.detach(|| drop(watcher.join())));
}
drop(self.wake.take());
if let Some(flags) = self.flags {
flags.default.store(true, Ordering::SeqCst);
flags.terminated.store(false, Ordering::SeqCst);
}
}
}

/// Where the watcher sends the interrupt: the main thread, and whether Python handles Ctrl+C there.
/// The signal the watcher sends the main thread: SIGINT where Python handles Ctrl+C, else the wake signal.
#[derive(Clone, Copy)]
struct MainThread {
ident: u64,
handles_interrupt: bool,
signal: Option<i64>,
}

/// Once SIGTERM arrives: let a second one end the process at once, cancel every running statement, and interrupt the main thread the way Ctrl+C does, where the platform can signal one thread.
Expand All @@ -119,9 +123,9 @@ fn watch(flags: &SigtermFlags, stop: &AtomicBool, main: MainThread) {
let cancelled = host_entry(py, ExecutorHost::CancelRunningStatements)
.and_then(|cancel| cancel.call0());
drop(cancelled);
if main.handles_interrupt {
if let Some(signal) = main.signal {
let interrupted = imported(py, "signal", "pthread_kill")
.and_then(|kill| kill.call1((main.ident, SIGINT_NUMBER)));
.and_then(|kill| kill.call1((main.ident, signal)));
drop(interrupted);
}
});
Expand All @@ -148,11 +152,14 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult<TerminationHandler> {
flags: None,
stop,
watcher: None,
wake: None,
});
};
let (signal, wake): (Option<i64>, Option<WakeHandler>) =
WakeHandler::interrupting_signal(py, || CommandTerminated::new_err(TERMINATED_MESSAGE))?;
let main = MainThread {
ident: native_field(&main_thread, "ident")?.extract()?,
handles_interrupt: action(SIGINT_NUMBER)?.is_callable(),
signal,
};
flags.terminated.store(false, Ordering::SeqCst);
let watched: Arc<AtomicBool> = Arc::clone(&stop);
Expand All @@ -164,6 +171,7 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult<TerminationHandler> {
flags: Some(flags),
stop,
watcher: Some(watcher),
wake,
})
}

Expand Down
7 changes: 6 additions & 1 deletion fensu.toml
Original file line number Diff line number Diff line change
Expand Up @@ -1655,6 +1655,11 @@ rule = "FFT104"
path = "tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py"
reason = "The disposable HTTPS gateway and OIDC provider dispatch real HTTP protocol messages and authentication transports; assertions in the parametrized installed journey remain branch-free."

[[targets.python.rule_exceptions]]
rule = "FFT104"
path = "tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py"
reason = "The interrupter thread polls for the event that starts the interrupt and must not interrupt once its block has ended; the tests using it remain branch-free."

[targets.rust]
analyzer = "rust"
roots = ["crates"]
Expand All @@ -1670,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X

[[targets.rust.threshold_overrides]]
paths = ["crates/sqlbuild-python/Cargo.toml"]
thresholds = { max_crate_code_lines = 66756 }
thresholds = { max_crate_code_lines = 66764 }
reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out."

[[targets.rust.threshold_overrides]]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,11 @@ can contain sensitive SQL, output and audit evidence.

A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it
cancels the statements it is running, records its real end time, its output and exit code 143,
prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process
at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run
stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end:
prints `error: terminated by SIGTERM`, and exits with code 143. It does this also when it was
started with SIGINT ignored, as a script starts a background job and some process managers start a
service; SIGINT and Ctrl-C then stay ignored. A second SIGTERM ends the process at once. Recall
keeps heartbeating while the command discovers and compiles the project, so a run stopped in any
phase is never mistaken for one that died. An `abandoned` run cannot record its end:
its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`.

## Selection
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,8 +215,7 @@ class CustomHostInterruptIntegrationTestCase:
description: str
model_count: int
hosts: int
interrupt_after_seconds: float
expected_max_seconds: float
expected_host_launches: int


@dataclass(frozen=True)
Expand Down
12 changes: 0 additions & 12 deletions tests/integration/src/sqlbuild/cli/commands/main/helpers.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
from __future__ import annotations

import _thread
import json
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import threading
import time
from collections.abc import Callable
from itertools import takewhile
Expand Down Expand Up @@ -920,16 +918,6 @@ def write_custom_host_project(*, project_dir: Path, model_count: int, rule_body:
)


def interrupt_main_after(delay_seconds: float) -> Callable[[Path], None]:
"""Return a host publish hook that interrupts the main thread after a delay."""

def interrupt(path: Path) -> None:
del path
threading.Timer(delay_seconds, _thread.interrupt_main).start()

return interrupt


def child_processes() -> frozenset[int]:
"""Return the live child processes of this process."""

Expand Down
42 changes: 26 additions & 16 deletions tests/integration/src/sqlbuild/cli/commands/main/test_rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,17 @@
child_processes,
compile_finding_keys,
compile_findings,
interrupt_main_after,
observe_hosts,
write_custom_host_project,
write_named_selector_project,
write_project_files,
write_project_scratch_rules,
write_python_rules_project,
)
from tests.unit.src.sqlbuild.rule_engine._helpers.host.interrupts.helpers import (
default_interrupt_handling,
interrupt_when,
)

_CACHE_RULES_TOML: str = (
'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n'
Expand Down Expand Up @@ -1388,8 +1391,7 @@ def failed_compile() -> tuple[int, dict[str, Any]]:
description="busy split hosts stop after an interrupt",
model_count=45,
hosts=3,
interrupt_after_seconds=1.5,
expected_max_seconds=12.0,
expected_host_launches=3,
),
],
ids=lambda case: case.description,
Expand All @@ -1399,26 +1401,34 @@ def test_given_busy_split_hosts_when_compile_is_interrupted_then_hosts_stop_and_
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
project_dir: Path = tmp_path / "orders"
project_dir.mkdir()
started: Path = tmp_path / "started"
started.mkdir()
write_custom_host_project(
project_dir=tmp_path,
project_dir=project_dir,
model_count=test_case.model_count,
rule_body="total = 0\nfor index in range(10_000_000):\n total += index\nreturn []",
rule_body=(
"from pathlib import Path\n"
f"(Path({str(started)!r}) / model.name).mkdir()\n"
"total = 0\nfor index in range(10_000_000):\n total += index\nreturn []"
),
)
children_before: frozenset[int] = child_processes()
hosts: CustomHostObserver = observe_hosts(
monkeypatch=monkeypatch,
cores=test_case.hosts,
publish_inputs=interrupt_main_after(test_case.interrupt_after_seconds),
)
started: float = time.monotonic()
hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=test_case.hosts)

with pytest.raises(KeyboardInterrupt):
_ = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"])
with (
default_interrupt_handling(),
interrupt_when(lambda: len(tuple(started.iterdir())) >= test_case.hosts) as sent,
pytest.raises(KeyboardInterrupt),
):
_ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"])

assert time.monotonic() - started < test_case.expected_max_seconds
assert len(hosts.launches()) == test_case.hosts
assert sent.is_set()
assert len(hosts.launches()) == test_case.expected_host_launches
assert len(tuple(started.iterdir())) < test_case.model_count
assert child_processes() <= children_before
assert not tuple((tmp_path / "target" / "rules-cache" / "host-inputs").iterdir())
assert not tuple((project_dir / "target" / "rules-cache" / "host-inputs").iterdir())


@pytest.mark.parametrize(
Expand Down
Loading
Loading