From 6bdc365aa814ee721bd3a81a5597c867a6738987 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:20:57 +0000 Subject: [PATCH 1/2] test: drive the interrupt and SIGTERM tests by events, not wall-clock time The busy-split-hosts interrupt tests and four Recall SIGTERM and Ctrl-C tests sent their signal on a timer and asserted elapsed time, so they failed in runners that start commands with SIGINT ignored or under load. They now send the signal once the blocking step has started (rule invocations started, the task printed, the store shows the step running) and assert outcomes and counts: the interrupt was raised, hosts launched, fewer invocations than models, no child processes or host inputs left, the exit code, no statement without an end, and that the step's post-sleep output is absent. A generous bound remains only to fail a hang. Processes the tests interrupt get default SIGINT handling. A new unit test interrupts the single-host fallback and fails if the pending interrupt is ever lost. --- fensu.toml | 5 + .../sqlbuild/cli/commands/main/_test_types.py | 3 +- .../src/sqlbuild/cli/commands/main/helpers.py | 12 -- .../sqlbuild/cli/commands/main/test_rules.py | 42 ++++--- .../src/sqlbuild/cli/recall/_test_types.py | 9 ++ .../src/sqlbuild/cli/recall/helpers.py | 107 +++++++++++------- .../recall/test_recall_replacement_sinks.py | 14 ++- .../cli/recall/test_recall_run_facts.py | 30 +++-- .../rule_engine/_helpers/host/_test_types.py | 7 +- .../rule_engine/_helpers/host/helpers.py | 24 ++-- .../_helpers/host/interrupts/helpers.py | 50 ++++++++ .../_helpers/host/test_custom_host_pool.py | 81 ++++++++++--- 12 files changed, 273 insertions(+), 111 deletions(-) create mode 100644 tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py diff --git a/fensu.toml b/fensu.toml index bb67834d42..b9a150746a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1655,6 +1655,11 @@ rule = "FFT104" path = "tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py" reason = "The disposable HTTPS gateway and OIDC provider dispatch real HTTP protocol messages and authentication transports; assertions in the parametrized installed journey remain branch-free." +[[targets.python.rule_exceptions]] +rule = "FFT104" +path = "tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py" +reason = "The interrupter thread polls for the event that starts the interrupt and must not interrupt once its block has ended; the tests using it remain branch-free." + [targets.rust] analyzer = "rust" roots = ["crates"] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index e2191ebab7..79e06bd725 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -215,8 +215,7 @@ class CustomHostInterruptIntegrationTestCase: description: str model_count: int hosts: int - interrupt_after_seconds: float - expected_max_seconds: float + expected_host_launches: int @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 44970058e9..d9a0ab9067 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1,6 +1,5 @@ from __future__ import annotations -import _thread import json import os import shutil @@ -8,7 +7,6 @@ import subprocess import sys import tempfile -import threading import time from collections.abc import Callable from itertools import takewhile @@ -920,16 +918,6 @@ def write_custom_host_project(*, project_dir: Path, model_count: int, rule_body: ) -def interrupt_main_after(delay_seconds: float) -> Callable[[Path], None]: - """Return a host publish hook that interrupts the main thread after a delay.""" - - def interrupt(path: Path) -> None: - del path - threading.Timer(delay_seconds, _thread.interrupt_main).start() - - return interrupt - - def child_processes() -> frozenset[int]: """Return the live child processes of this process.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index fce0de2a8f..9662e05e3a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -53,7 +53,6 @@ child_processes, compile_finding_keys, compile_findings, - interrupt_main_after, observe_hosts, write_custom_host_project, write_named_selector_project, @@ -61,6 +60,10 @@ write_project_scratch_rules, write_python_rules_project, ) +from tests.unit.src.sqlbuild.rule_engine._helpers.host.interrupts.helpers import ( + default_interrupt_handling, + interrupt_when, +) _CACHE_RULES_TOML: str = ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n' @@ -1388,8 +1391,7 @@ def failed_compile() -> tuple[int, dict[str, Any]]: description="busy split hosts stop after an interrupt", model_count=45, hosts=3, - interrupt_after_seconds=1.5, - expected_max_seconds=12.0, + expected_host_launches=3, ), ], ids=lambda case: case.description, @@ -1399,26 +1401,34 @@ def test_given_busy_split_hosts_when_compile_is_interrupted_then_hosts_stop_and_ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + project_dir: Path = tmp_path / "orders" + project_dir.mkdir() + started: Path = tmp_path / "started" + started.mkdir() write_custom_host_project( - project_dir=tmp_path, + project_dir=project_dir, model_count=test_case.model_count, - rule_body="total = 0\nfor index in range(10_000_000):\n total += index\nreturn []", + rule_body=( + "from pathlib import Path\n" + f"(Path({str(started)!r}) / model.name).mkdir()\n" + "total = 0\nfor index in range(10_000_000):\n total += index\nreturn []" + ), ) children_before: frozenset[int] = child_processes() - hosts: CustomHostObserver = observe_hosts( - monkeypatch=monkeypatch, - cores=test_case.hosts, - publish_inputs=interrupt_main_after(test_case.interrupt_after_seconds), - ) - started: float = time.monotonic() + hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=test_case.hosts) - with pytest.raises(KeyboardInterrupt): - _ = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + with ( + default_interrupt_handling(), + interrupt_when(lambda: len(tuple(started.iterdir())) >= test_case.hosts) as sent, + pytest.raises(KeyboardInterrupt), + ): + _ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) - assert time.monotonic() - started < test_case.expected_max_seconds - assert len(hosts.launches()) == test_case.hosts + assert sent.is_set() + assert len(hosts.launches()) == test_case.expected_host_launches + assert len(tuple(started.iterdir())) < test_case.model_count assert child_processes() <= children_before - assert not tuple((tmp_path / "target" / "rules-cache" / "host-inputs").iterdir()) + assert not tuple((project_dir / "target" / "rules-cache" / "host-inputs").iterdir()) @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index e161327af8..8b980d6c75 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -198,6 +198,14 @@ class SlowReplacementTestCase: expected_max_seconds: float +@dataclass(frozen=True) +class InterruptedSinkTestCase: + description: str + shutdown_seconds: float + sink_sleep_seconds: str + expected_exit_code: int + + @dataclass(frozen=True) class ThirdPartyLogTestCase: description: str @@ -369,6 +377,7 @@ class TerminatedCommandTestCase: environment: dict[str, str] wait_sql: str expected_output: bytes + unfinished_output: bytes @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 569ed78183..21c9158c9c 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -26,6 +26,8 @@ MEBIBYTE: int = 1024 * 1024 HANG_SECONDS: int = 600 HANG_DUMP_GRACE_SECONDS: int = 30 +# Only a hang takes this long: a signalled build stops within a second or two. +STOP_HANG_SECONDS: int = 120 EMPTY_RESULT: bytes = b'{"rows": []}' NUMBERED_LINE: re.Pattern[bytes] = re.compile(rb"^(\d+) std(?:out|err)$", re.MULTILINE) PROJECT_TOML: str = ( @@ -61,6 +63,7 @@ f" sys.stdout.write({EMITTED_TAIL.decode()!r})\n" " sys.stdout.flush()\n" ' time.sleep(float(os.environ.get("ORDERS_TASK_SLEEP_SECONDS", "0")))\n' + ' sys.stdout.write(os.environ.get("ORDERS_TASK_AFTER_SLEEP", ""))\n' ' _write_concurrently(int(os.environ.get("ORDERS_TASK_THREAD_LINES", "0")))\n' ' sys.stdout.write("y" * int(os.environ.get("ORDERS_TASK_STDOUT_BYTES", "0")))\n' " sys.stdout.flush()\n" @@ -118,7 +121,7 @@ def environment(**overrides: str) -> dict[str, str]: def run(project_dir: Path, *args: str, **env: str) -> subprocess.CompletedProcess[bytes]: """Run the Python CLI once in the project and capture exact streams, or fail with its stacks.""" - process: subprocess.Popen[bytes] = start(project_dir, *args, **env) + process: subprocess.Popen[bytes] = start(project_dir, *args, interrupts=signal.SIG_DFL, **env) try: stdout, stderr = process.communicate(timeout=HANG_SECONDS) except subprocess.TimeoutExpired: @@ -142,8 +145,15 @@ def hang_dump(process: subprocess.Popen[bytes]) -> str: return stderr.decode(errors="replace") -def start(project_dir: Path, *args: str, **env: str) -> subprocess.Popen[bytes]: - """Start the Python CLI in its own process group.""" +def start( + project_dir: Path, + *args: str, + interrupts: signal.Handlers = signal.SIG_DFL, + **env: str, +) -> subprocess.Popen[bytes]: + """Start the Python CLI in its own process group with SIGINT set to `interrupts`. + + The default gives the CLI Ctrl-C handling even when this runner ignores SIGINT.""" return subprocess.Popen( [str(PYTHON_CLI), *args], @@ -152,6 +162,7 @@ def start(project_dir: Path, *args: str, **env: str) -> subprocess.Popen[bytes]: stderr=subprocess.PIPE, env=environment(**env), start_new_session=True, + preexec_fn=lambda: signal.signal(signal.SIGINT, interrupts), ) @@ -251,36 +262,60 @@ def write_slow_import_sink(project_dir: Path) -> Path: sink: Path = project_dir / "sinks" / "slow_import.py" sink.parent.mkdir(parents=True, exist_ok=True) _ = sink.write_text( - "import os\nimport time\n\n" - "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n", + "import os\nimport sys\nimport time\n\n" + "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n" + "sys.stdout.write(os.environ.get('ORDERS_IMPORT_AFTER_SLEEP', ''))\n", encoding="utf-8", ) return project_dir def terminated_build( - project_dir: Path, *, wait_sql: str, **env: str -) -> tuple[subprocess.CompletedProcess[bytes], float]: - """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row, and the - seconds it took to stop.""" + project_dir: Path, + *, + wait_sql: str, + **env: str, +) -> subprocess.CompletedProcess[bytes]: + """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row.""" process: subprocess.Popen[bytes] = start(project_dir, "build", **env) try: _ = wait_for(wait_sql, project_dir, timeout=120) process.send_signal(signal.SIGTERM) - sent: float = time.monotonic() - stdout, stderr = process.communicate(timeout=60) + stdout, stderr = process.communicate(timeout=STOP_HANG_SECONDS) except subprocess.TimeoutExpired: - raise AssertionError(f"a terminated build kept running:\n{hang_dump(process)}") from None - completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( - process.args, process.returncode, stdout, stderr + raise AssertionError(f"a terminated build hung:\n{hang_dump(process)}") from None + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + + +def main_thread_state(pid: int) -> tuple[str, str, str]: + """The scheduler state and user and system CPU ticks of the process's main thread.""" + + fields: list[str] = ( + Path(f"/proc/{pid}/task/{pid}/stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split() + ) + return fields[0], fields[11], fields[12] + + +def wait_until_blocked(pid: int, *, samples: int = 5) -> None: + """Wait until the main thread has slept without using CPU for `samples` polls in a row. + + A thread waiting for the CPU reports `R`, so load does not pass for blocked.""" + + seen: list[tuple[str, str, str]] = [main_thread_state(pid)] + deadline: float = time.monotonic() + STOP_HANG_SECONDS + while (seen[-samples:] != [seen[-1]] * samples or seen[-1][0] != "S") and ( + time.monotonic() < deadline + ): + time.sleep(0.1) + seen.append(main_thread_state(pid)) + assert seen[-samples:] == [seen[-1]] * samples and seen[-1][0] == "S", ( + f"the build's main thread never blocked, so it hung: {seen[-samples:]}" ) - return completed, time.monotonic() - sent -def terminated_while_closing(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float]: - """A build sent SIGTERM while it waits for a locked Recall store at exit, and the seconds it - took to stop.""" +def terminated_while_closing(project_dir: Path) -> subprocess.CompletedProcess[bytes]: + """A build sent SIGTERM while it waits for a locked Recall store at exit.""" with held_store_lock(project_dir): process: subprocess.Popen[bytes] = start(project_dir, "build", PYTHONUNBUFFERED="1") @@ -288,36 +323,32 @@ def terminated_while_closing(project_dir: Path) -> tuple[subprocess.CompletedPro printed: bytes = b"" while b"Completed" not in printed and process.poll() is None: printed += process.stdout.readline() - time.sleep(1) + wait_until_blocked(process.pid) process.send_signal(signal.SIGTERM) - sent: float = time.monotonic() try: - stdout, stderr = process.communicate(timeout=60) + stdout, stderr = process.communicate(timeout=STOP_HANG_SECONDS) except subprocess.TimeoutExpired: - raise AssertionError(f"a closing build kept running:\n{hang_dump(process)}") from None - completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( - process.args, process.returncode, printed + stdout, stderr - ) - return completed, time.monotonic() - sent + raise AssertionError(f"a closing build hung:\n{hang_dump(process)}") from None + return subprocess.CompletedProcess(process.args, process.returncode, printed + stdout, stderr) -def interrupted_build(project_dir: Path, **env: str) -> tuple[int | None, float]: - """Exit code of a build sent SIGINT after its first logged event, and seconds to stop.""" +def interrupted_build(project_dir: Path, **env: str) -> int | None: + """Exit code of a build sent SIGINT once its task has printed; `None` when it hung.""" - log: Path = project_dir / "events.log" - process: subprocess.Popen[bytes] = start(project_dir, "build", **env) + process: subprocess.Popen[bytes] = start( + project_dir, "build", interrupts=signal.SIG_DFL, PYTHONUNBUFFERED="1", **env + ) try: - deadline: float = time.monotonic() + 120 - while not log.exists() and time.monotonic() < deadline: - time.sleep(0.1) - time.sleep(1) - interrupted: float = time.monotonic() + assert process.stdout is not None + printed: bytes = b"" + while EMITTED_TAIL not in printed and process.poll() is None: + printed += process.stdout.readline() os.killpg(process.pid, signal.SIGINT) try: - _ = process.communicate(timeout=30) + _ = process.communicate(timeout=STOP_HANG_SECONDS) except subprocess.TimeoutExpired: - return None, time.monotonic() - interrupted - return process.returncode, time.monotonic() - interrupted + return None + return process.returncode finally: with contextlib.suppress(ProcessLookupError): os.killpg(process.pid, signal.SIGKILL) diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py index f993680526..1d0df8d2f3 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py @@ -1,5 +1,6 @@ from __future__ import annotations +import signal import subprocess import time from pathlib import Path @@ -9,6 +10,7 @@ from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.integration.src.sqlbuild.cli.recall._test_types import ( FailingReplacementTestCase, + InterruptedSinkTestCase, ReplacementLaunchRecordTestCase, ReplacementLifecycleTestCase, ReplacementOutputTestCase, @@ -224,17 +226,17 @@ def test_given_a_lifecycle_sink_slower_than_the_budget_when_building_then_nothin @pytest.mark.parametrize( "test_case", [ - SlowReplacementTestCase( + InterruptedSinkTestCase( description="Ctrl-C stops the build while the sink sleeps", shutdown_seconds=1.0, sink_sleep_seconds="30", - expected_max_seconds=10.0, + expected_exit_code=-signal.SIGINT, ), ], ids=lambda case: case.description, ) def test_given_a_slow_lifecycle_sink_when_interrupted_then_the_build_stops( - test_case: SlowReplacementTestCase, tmp_path: Path + test_case: InterruptedSinkTestCase, tmp_path: Path ) -> None: budget: str = f'\n[recall]\nshutdown_timeout = "{int(test_case.shutdown_seconds * 1000)}ms"\n' project_dir: Path = write_sinks( @@ -242,14 +244,14 @@ def test_given_a_slow_lifecycle_sink_when_interrupted_then_the_build_stops( {"recall_events": SLOW_LIFECYCLE_SINK, "event_log": EVENT_LOG_SINK}, ) - exit_code, stopped_after = interrupted_build( + exit_code: int | None = interrupted_build( project_dir, ORDERS_SINK_SLEEP_SECONDS=test_case.sink_sleep_seconds, ORDERS_TASK_SLEEP_SECONDS="60", ) - assert exit_code not in (None, 0) - assert stopped_after < test_case.expected_max_seconds + assert exit_code is not None, "the interrupted build hung" + assert exit_code == test_case.expected_exit_code if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py index 42ca4741b9..fdb8555776 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -90,21 +90,27 @@ def test_given_selectors_when_building_then_the_effective_selection_is_recorded( TerminatedCommandTestCase( description="during project discovery", orders_model=ORDERS_MODEL, - environment={"ORDERS_IMPORT_SECONDS": "60"}, + environment={ + "ORDERS_IMPORT_SECONDS": "60", + "ORDERS_IMPORT_AFTER_SLEEP": "slow sink imported\n", + }, wait_sql=( "SELECT invocation_id FROM invocations WHERE status = 'running' " "AND (julianday(last_heartbeat_at) - julianday(started_at)) * 86400 >= 2" ), expected_output=b"", + unfinished_output=b"slow sink imported", ), TerminatedCommandTestCase( description="while a task runs", orders_model=ORDERS_MODEL, - environment={"ORDERS_TASK_SLEEP_SECONDS": "60"}, - wait_sql=( - "SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'" - ), + environment={ + "ORDERS_TASK_SLEEP_SECONDS": "60", + "ORDERS_TASK_AFTER_SLEEP": "slept through\n", + }, + wait_sql="SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'", expected_output=EMITTED_TAIL, + unfinished_output=b"slept through", ), TerminatedCommandTestCase( description="while a long warehouse statement runs", @@ -118,6 +124,7 @@ def test_given_selectors_when_building_then_the_effective_selection_is_recorded( "WHERE statement_kind = 'CREATE' AND ended_at IS NULL" ), expected_output=b"", + unfinished_output=b"Completed", ), ], ids=lambda case: case.description, @@ -128,7 +135,7 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit project_dir: Path = write_slow_import_sink(write_project(tmp_path / "orders_shop")) _ = (project_dir / "models" / "orders.sql").write_text(test_case.orders_model, encoding="utf-8") - terminated, stop_seconds = terminated_build( + terminated: subprocess.CompletedProcess[bytes] = terminated_build( project_dir, wait_sql=test_case.wait_sql, SQLBUILD_RECALL_HEARTBEAT_SECONDS="0.5", @@ -143,10 +150,14 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit starts: list[list[object]] = recall_rows( project_dir, "SELECT COUNT(*) FROM recall_events WHERE event_type = 'invocation_started'" ) + running_statements: list[list[object]] = recall_rows( + project_dir, "SELECT COUNT(*) FROM statements WHERE ended_at IS NULL" + ) output: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "output", "last") assert terminated.returncode == 143 - assert stop_seconds < 10 + assert test_case.unfinished_output not in terminated.stdout + output.stdout + assert running_statements == [[0]] assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") assert b"Traceback" not in terminated.stderr assert recorded == [["cancelled", 15, 143, 0, 1, 1]] @@ -160,7 +171,7 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit [ ClosingTerminationTestCase( description="a build waiting for a locked store at exit", - shutdown_timeout="60s", + shutdown_timeout="600s", expected_exit_code=143, ), ], @@ -174,13 +185,12 @@ def test_given_sigterm_while_recall_closes_when_a_build_has_finished_then_it_exi extra_toml=f'\n[recall]\nshutdown_timeout = "{test_case.shutdown_timeout}"\n', ) - terminated, stop_seconds = terminated_while_closing(project_dir) + terminated: subprocess.CompletedProcess[bytes] = terminated_while_closing(project_dir) assert b"Completed" in terminated.stdout assert terminated.returncode == test_case.expected_exit_code, terminated.stderr.decode( errors="replace" ) - assert stop_seconds < 10 assert b"Traceback" not in terminated.stderr assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py index c0d9afb685..c18bbe10dc 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/_test_types.py @@ -40,14 +40,15 @@ class HostCancellationTestCase: @dataclass(frozen=True) class HostInterruptTestCase: - """An interrupt while split hosts are still busy with slow invocations.""" + """An interrupt sent once `interrupt_at_hosts` host processes are busy with slow invocations.""" description: str model_count: int hosts: int delay_seconds: float - interrupt_after_seconds: float - expected_max_seconds: float + interrupt_at_hosts: int + expected_host_launches: int + failing: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index 8fe68caed4..c6bf7c5a0d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -1,10 +1,8 @@ """Builders for custom-rule host partitioning tests.""" -import _thread import os import random import shutil -import threading import time from collections.abc import Callable from operator import attrgetter @@ -38,6 +36,8 @@ @rule(code="{_MODEL_RULE}", message="order models need an even suffix", remediation="Rename it.") def even_suffix(*, model: Model, ctx: RuleContext) -> list[Finding]: + if STARTED_DIR: + os.mkdir(Path(STARTED_DIR) / f"{{model.name}}.{{os.getpid()}}") if model.name in FAILING: raise ValueError(f"cannot inspect {{model.name}}") time.sleep(DELAY_SECONDS) @@ -68,13 +68,19 @@ def write_order_rules( records_module_state: bool = False, detects_duplicates: bool = False, delay_seconds: float = 0.0, + started_dir: Path | str = "", ) -> None: - """Write one model rule and one project rule with optional failures, state, or delays.""" + """Write one model rule and one project rule with optional failures, state, or delays. + + With `started_dir`, each model invocation first creates the folder `.` there; + the host's hermeticity guard refuses opening files but allows creating folders.""" rules: Path = root / "rules" / "orders.py" rules.parent.mkdir(parents=True, exist_ok=True) rules.write_text( - f"import time\n\nFAILING = {set(failing)!r}\nRECORD_SEEN = {records_module_state!r}\n" + "import os\nimport time\nfrom pathlib import Path\n\n" + f"STARTED_DIR = {str(started_dir)!r}\n" + f"FAILING = {set(failing)!r}\nRECORD_SEEN = {records_module_state!r}\n" f"DETECT_DUPLICATES = {detects_duplicates!r}\nDELAY_SECONDS = {delay_seconds!r}\n" f"SEEN: dict[str, bool] = {{}}\n{_RULES_MODULE}", encoding="utf-8", @@ -188,14 +194,10 @@ def fail(path: Path) -> None: return fail, failures -def interrupting_publish(delay_seconds: float) -> Callable[[Path], None]: - """Return a hook that interrupts the main thread once hosts have run for a while.""" - - def interrupt(path: Path) -> None: - del path - threading.Timer(delay_seconds, _thread.interrupt_main).start() +def started_hosts(started_dir: Path) -> int: + """Return how many host processes started an invocation, from `.` folders.""" - return interrupt + return len({path.suffix for path in started_dir.iterdir()}) def record_signals(*, monkeypatch: pytest.MonkeyPatch) -> list[tuple[int, int]]: diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py new file mode 100644 index 0000000000..3374193072 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/interrupts/helpers.py @@ -0,0 +1,50 @@ +"""Interrupt the main thread once an observed event happens, the way Ctrl-C does.""" + +import _thread +import contextlib +import signal +import threading +from collections.abc import Callable, Iterator + + +@contextlib.contextmanager +def default_interrupt_handling() -> Iterator[None]: + """Give the main thread Python's default Ctrl-C handling while the block runs. + + Test runners often start with SIGINT ignored; `_thread.interrupt_main` then does nothing.""" + + previous: object = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + yield + finally: + _ = signal.signal(signal.SIGINT, previous) + + +@contextlib.contextmanager +def interrupt_when(ready: Callable[[], bool]) -> Iterator[threading.Event]: + """Interrupt the main thread once `ready()` holds. + + The event is set when the interrupt was sent; leaving the block stops the watcher, so no + interrupt arrives after it.""" + + sent: threading.Event = threading.Event() + stop: threading.Event = threading.Event() + lock: threading.Lock = threading.Lock() + + def watch() -> None: + while not stop.wait(0.02): + if ready(): + with lock: + if not stop.is_set(): + sent.set() + _thread.interrupt_main() + return + + watcher: threading.Thread = threading.Thread(target=watch, daemon=True) + watcher.start() + try: + yield sent + finally: + with lock: + stop.set() + watcher.join() diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 5ba1f73053..6569d66040 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -30,13 +30,17 @@ failed_publish, failed_publish_without_directory, host_inputs_left, - interrupting_publish, orders_project, record_signals, + started_hosts, touch_files, use_hosts, write_order_rules, ) +from tests.unit.src.sqlbuild.rule_engine._helpers.host.interrupts.helpers import ( + default_interrupt_handling, + interrupt_when, +) @pytest.mark.parametrize( @@ -233,8 +237,8 @@ def test_given_failing_host_when_others_are_busy_then_they_stop_before_later_inv model_count=60, hosts=3, delay_seconds=1.0, - interrupt_after_seconds=1.5, - expected_max_seconds=10.0, + interrupt_at_hosts=3, + expected_host_launches=3, ), ], ids=lambda case: case.description, @@ -242,25 +246,76 @@ def test_given_failing_host_when_others_are_busy_then_they_stop_before_later_inv def test_given_busy_split_hosts_when_interrupted_then_hosts_stop_and_no_process_remains( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostInterruptTestCase ) -> None: - write_order_rules(root=tmp_path, delay_seconds=test_case.delay_seconds) + started: Path = tmp_path / "started" + started.mkdir() + write_order_rules(root=tmp_path, delay_seconds=test_case.delay_seconds, started_dir=started) project: CompiledProjectView = orders_project(model_count=test_case.model_count) children_before: frozenset[int] = child_processes() - hosts: CustomHostObserver = use_hosts( - monkeypatch=monkeypatch, - hosts=test_case.hosts, - publish_inputs=interrupting_publish(test_case.interrupt_after_seconds), - ) - started: float = time.monotonic() + hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) - with pytest.raises(KeyboardInterrupt): + with ( + default_interrupt_handling(), + interrupt_when(lambda: started_hosts(started) >= test_case.interrupt_at_hosts) as sent, + pytest.raises(KeyboardInterrupt), + ): _ = evaluate_order_rules(project=project, root=tmp_path) - assert time.monotonic() - started < test_case.expected_max_seconds - assert len(hosts.launches()) == test_case.hosts + assert sent.is_set() + assert len(hosts.launches()) == test_case.expected_host_launches + assert len(tuple(started.iterdir())) < test_case.model_count assert child_processes() <= children_before assert not host_inputs_left(tmp_path) +@pytest.mark.parametrize( + "test_case", + [ + HostInterruptTestCase( + description="interrupt while the single-host fallback runs", + model_count=30, + hosts=3, + delay_seconds=0.2, + interrupt_at_hosts=4, + expected_host_launches=4, + failing=("orders_029",), + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupt_while_hosts_are_busy_when_the_run_ends_then_the_interrupt_is_raised( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostInterruptTestCase +) -> None: + started: Path = tmp_path / "started" + started.mkdir() + write_order_rules( + root=tmp_path, + delay_seconds=test_case.delay_seconds, + started_dir=started, + failing=test_case.failing, + ) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) + hosts: CustomHostObserver = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) + outcomes: list[str] = [] + + with default_interrupt_handling(): + try: + with interrupt_when( + lambda: started_hosts(started) >= test_case.interrupt_at_hosts + ) as sent: + try: + _ = evaluate_order_rules(project=project, root=tmp_path, cache_enabled=False) + except RulesError: + outcomes.append("RulesError") + for _ in range(100): + time.sleep(0.01) # a pending interrupt is raised between bytecodes + except KeyboardInterrupt: + outcomes.append("KeyboardInterrupt") + + assert sent.is_set() + assert len(hosts.launches()) == test_case.expected_host_launches + assert outcomes[-1:] == ["KeyboardInterrupt"] + + @pytest.mark.parametrize( "test_case", ( From c17c972e07fb24243666b9b5ca9c3301cd0a940a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:21:06 +0000 Subject: [PATCH 2/2] fix: stop on SIGTERM when the process was started with SIGINT ignored After SIGTERM, the command cancels its running statements and interrupts its main thread by sending it SIGINT, but only when Python handles SIGINT. A process started with SIGINT ignored, as background jobs of scripts and many process managers and CI runners are, therefore kept running its blocking step, such as a task or a slow sink import, to the end. Orchestrators then SIGKILL it after their grace period, before it cleans up. When SIGINT is not handled, the command now installs a handler on an otherwise unused signal (SIGRTMIN, or SIGUSR2 where there is none) while that signal is at its default action, and the SIGTERM watcher interrupts the main thread with it. The handler raises the same termination error, so the command exits 143 as before, and is restored when the command ends. SIGINT itself is unchanged: when it was ignored, Ctrl-C stays ignored. Windows is unchanged. An end-to-end test starts the native sqb build with SIGINT ignored and sends SIGTERM once its task is running. --- .../src/native_host/_helpers/interrupt.rs | 18 +++- .../src/native_host/models.rs | 85 ++++++++++++++++--- .../src/bindings/_helpers/cli/termination.rs | 20 +++-- fensu.toml | 2 +- .../docs/concepts/observability/recall.md | 8 +- .../src/sqlbuild/cli/recall/_test_types.py | 5 ++ .../src/sqlbuild/cli/recall/helpers.py | 22 +++-- .../cli/recall/test_recall_run_facts.py | 17 ++++ .../docs/concepts/observability/recall.mdx | 8 +- 9 files changed, 155 insertions(+), 30 deletions(-) diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs index ca192c2c4e..e2b6d9e56d 100644 --- a/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/interrupt.rs @@ -3,7 +3,7 @@ use std::cell::Cell; use pyo3::exceptions::PyKeyboardInterrupt; -use pyo3::prelude::{PyResult, Python}; +use pyo3::prelude::{Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; thread_local! { pub(crate) static INTERRUPTED: Cell = const { Cell::new(false) }; @@ -23,3 +23,19 @@ pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> { } Ok(()) } + +/// Put `signal`'s previous Python handler back, unless something replaced `installed` meanwhile. Fails when a pending handler raised first, as `signal.signal` runs pending handlers before it changes one. +pub(crate) fn restore_signal_handler( + py: Python<'_>, + signal: i64, + (installed, previous): (&Py, &Py), +) -> PyResult<()> { + let module: pyo3::Bound<'_, PyModule> = py.import("signal")?; + if module + .call_method1("getsignal", (signal,))? + .is(installed.bind(py)) + { + module.call_method1("signal", (signal, previous.bind(py)))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-base/src/native_host/models.rs b/crates/sqlbuild-python-base/src/native_host/models.rs index 32544d9016..8e6757a6fa 100644 --- a/crates/sqlbuild-python-base/src/native_host/models.rs +++ b/crates/sqlbuild-python-base/src/native_host/models.rs @@ -2,11 +2,13 @@ use crate::native_host::_helpers::interrupt::INTERRUPTED; use crate::native_host::_helpers::interrupt::SIGINT; +use crate::native_host::_helpers::interrupt::restore_signal_handler; use pyo3::exceptions::PyKeyboardInterrupt; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::sync::PyOnceLock; @@ -107,22 +109,83 @@ impl InterruptHandler { impl Drop for InterruptHandler { fn drop(&mut self) { Python::attach(|py| { - let restored: PyResult<()> = (|| { - let signal = py.import("signal")?; - if signal - .call_method1("getsignal", (SIGINT,))? - .is(self.installed.bind(py)) - { - signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?; - } - Ok(()) - })(); - drop(restored); + drop(restore_signal_handler( + py, + SIGINT, + (&self.installed, &self.previous), + )); }); INTERRUPTED.set(false); } } +/// A command's handler on an otherwise unused signal that raises the command's error, so another thread can interrupt the main thread while SIGINT is ignored, without changing how SIGINT is handled. +#[derive(Debug)] +pub struct WakeHandler { + pub signal: i64, + pub(crate) previous: Py, + pub(crate) installed: Py, +} + +impl WakeHandler { + /// The signal that interrupts the main thread: SIGINT where Python handles it, else the signal of a wake handler raising `error`, installed here; `None` when neither is available. + pub fn interrupting_signal( + py: Python<'_>, + error: fn() -> PyErr, + ) -> PyResult<(Option, Option)> { + let signal = py.import("signal")?; + if signal.call_method1("getsignal", (SIGINT,))?.is_callable() { + return Ok((Some(SIGINT), None)); + } + let wake: Option = Self::install(py, error)?; + Ok((wake.as_ref().map(|wake| wake.signal), wake)) + } + + /// Install the handler on SIGRTMIN, or SIGUSR2 where there is none, while that signal takes its default action. + fn install(py: Python<'_>, error: fn() -> PyErr) -> PyResult> { + let signal = py.import("signal")?; + let name: &str = if signal.hasattr("SIGRTMIN")? { + "SIGRTMIN" + } else if signal.hasattr("SIGUSR2")? { + "SIGUSR2" + } else { + return Ok(None); + }; + let number: i64 = signal.getattr(name)?.extract()?; + let previous = signal.call_method1("getsignal", (number,))?; + if !previous.is(signal.getattr("SIG_DFL")?) { + return Ok(None); + } + let installed = PyCFunction::new_closure( + py, + None, + None, + move |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + Err(error()) + }, + )?; + signal.call_method1("signal", (number, &installed))?; + Ok(Some(Self { + signal: number, + previous: previous.unbind(), + installed: installed.into_any().unbind(), + })) + } +} + +impl Drop for WakeHandler { + /// Restore the previous handler. A wake still pending raises on the first attempt and is consumed, so the second attempt restores. + fn drop(&mut self) { + Python::attach(|py| { + let handlers = (&self.installed, &self.previous); + drop( + restore_signal_handler(py, self.signal, handlers) + .or_else(|_pending| restore_signal_handler(py, self.signal, handlers)), + ); + }); + } +} + impl std::fmt::Debug for AuditProjection { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs index 512886cef8..ab7d77a9f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs @@ -16,6 +16,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::WakeHandler; use sqlbuild_python_base::records::main::fields::field::field as native_field; create_exception!( @@ -69,11 +70,12 @@ fn registered_flags() -> Option<&'static SigtermFlags> { .as_ref() } -/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. Holds only Rust values: while it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again. +/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. While it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again. pub(crate) struct TerminationHandler { flags: Option<&'static SigtermFlags>, stop: Arc, watcher: Option>, + wake: Option, } impl TerminationHandler { @@ -86,6 +88,7 @@ impl TerminationHandler { if let Some(watcher) = self.watcher.take() { py.detach(|| drop(watcher.join())); } + drop(self.wake.take()); terminated() } } @@ -96,6 +99,7 @@ impl Drop for TerminationHandler { if let Some(watcher) = self.watcher.take() { Python::attach(|py| py.detach(|| drop(watcher.join()))); } + drop(self.wake.take()); if let Some(flags) = self.flags { flags.default.store(true, Ordering::SeqCst); flags.terminated.store(false, Ordering::SeqCst); @@ -103,11 +107,11 @@ impl Drop for TerminationHandler { } } -/// Where the watcher sends the interrupt: the main thread, and whether Python handles Ctrl+C there. +/// The signal the watcher sends the main thread: SIGINT where Python handles Ctrl+C, else the wake signal. #[derive(Clone, Copy)] struct MainThread { ident: u64, - handles_interrupt: bool, + signal: Option, } /// Once SIGTERM arrives: let a second one end the process at once, cancel every running statement, and interrupt the main thread the way Ctrl+C does, where the platform can signal one thread. @@ -119,9 +123,9 @@ fn watch(flags: &SigtermFlags, stop: &AtomicBool, main: MainThread) { let cancelled = host_entry(py, ExecutorHost::CancelRunningStatements) .and_then(|cancel| cancel.call0()); drop(cancelled); - if main.handles_interrupt { + if let Some(signal) = main.signal { let interrupted = imported(py, "signal", "pthread_kill") - .and_then(|kill| kill.call1((main.ident, SIGINT_NUMBER))); + .and_then(|kill| kill.call1((main.ident, signal))); drop(interrupted); } }); @@ -148,11 +152,14 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult { flags: None, stop, watcher: None, + wake: None, }); }; + let (signal, wake): (Option, Option) = + WakeHandler::interrupting_signal(py, || CommandTerminated::new_err(TERMINATED_MESSAGE))?; let main = MainThread { ident: native_field(&main_thread, "ident")?.extract()?, - handles_interrupt: action(SIGINT_NUMBER)?.is_callable(), + signal, }; flags.terminated.store(false, Ordering::SeqCst); let watched: Arc = Arc::clone(&stop); @@ -164,6 +171,7 @@ pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult { flags: Some(flags), stop, watcher: Some(watcher), + wake, }) } diff --git a/fensu.toml b/fensu.toml index b9a150746a..58c48e91b3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1675,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 66709 } +thresholds = { max_crate_code_lines = 66717 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index bfeaf3eb62..dd4179906a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -155,9 +155,11 @@ can contain sensitive SQL, output and audit evidence. A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it cancels the statements it is running, records its real end time, its output and exit code 143, -prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process -at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run -stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +prints `error: terminated by SIGTERM`, and exits with code 143. It does this also when it was +started with SIGINT ignored, as a script starts a background job and some process managers start a +service; SIGINT and Ctrl-C then stay ignored. A second SIGTERM ends the process at once. Recall +keeps heartbeating while the command discovers and compiles the project, so a run stopped in any +phase is never mistaken for one that died. An `abandoned` run cannot record its end: its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. ## Selection diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 8b980d6c75..c07441efee 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -1,7 +1,10 @@ +import signal from collections.abc import Callable from dataclasses import dataclass from pathlib import Path +from tests.integration.src.sqlbuild.cli.recall.helpers import PYTHON_CLI + @dataclass(frozen=True) class RecallOutputTestCase: @@ -378,6 +381,8 @@ class TerminatedCommandTestCase: wait_sql: str expected_output: bytes unfinished_output: bytes + interrupts: signal.Handlers = signal.SIG_DFL + program: Path = PYTHON_CLI @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 21c9158c9c..d88e972708 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -121,7 +121,9 @@ def environment(**overrides: str) -> dict[str, str]: def run(project_dir: Path, *args: str, **env: str) -> subprocess.CompletedProcess[bytes]: """Run the Python CLI once in the project and capture exact streams, or fail with its stacks.""" - process: subprocess.Popen[bytes] = start(project_dir, *args, interrupts=signal.SIG_DFL, **env) + process: subprocess.Popen[bytes] = start( + project_dir, *args, interrupts=signal.SIG_DFL, program=PYTHON_CLI, **env + ) try: stdout, stderr = process.communicate(timeout=HANG_SECONDS) except subprocess.TimeoutExpired: @@ -149,14 +151,15 @@ def start( project_dir: Path, *args: str, interrupts: signal.Handlers = signal.SIG_DFL, + program: Path = PYTHON_CLI, **env: str, ) -> subprocess.Popen[bytes]: - """Start the Python CLI in its own process group with SIGINT set to `interrupts`. + """Start `program` in its own process group with SIGINT set to `interrupts`. The default gives the CLI Ctrl-C handling even when this runner ignores SIGINT.""" return subprocess.Popen( - [str(PYTHON_CLI), *args], + [str(program), *args], cwd=project_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -274,11 +277,15 @@ def terminated_build( project_dir: Path, *, wait_sql: str, + interrupts: signal.Handlers = signal.SIG_DFL, + program: Path = PYTHON_CLI, **env: str, ) -> subprocess.CompletedProcess[bytes]: """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row.""" - process: subprocess.Popen[bytes] = start(project_dir, "build", **env) + process: subprocess.Popen[bytes] = start( + project_dir, "build", interrupts=interrupts, program=program, **env + ) try: _ = wait_for(wait_sql, project_dir, timeout=120) process.send_signal(signal.SIGTERM) @@ -336,7 +343,12 @@ def interrupted_build(project_dir: Path, **env: str) -> int | None: """Exit code of a build sent SIGINT once its task has printed; `None` when it hung.""" process: subprocess.Popen[bytes] = start( - project_dir, "build", interrupts=signal.SIG_DFL, PYTHONUNBUFFERED="1", **env + project_dir, + "build", + interrupts=signal.SIG_DFL, + program=PYTHON_CLI, + PYTHONUNBUFFERED="1", + **env, ) try: assert process.stdout is not None diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py index fdb8555776..0cfcd34df3 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -1,4 +1,5 @@ import json +import signal import subprocess from pathlib import Path @@ -15,6 +16,7 @@ ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( EMITTED_TAIL, + NATIVE_SQB, ORDERS_MODEL, recall_rows, run, @@ -112,6 +114,19 @@ def test_given_selectors_when_building_then_the_effective_selection_is_recorded( expected_output=EMITTED_TAIL, unfinished_output=b"slept through", ), + TerminatedCommandTestCase( + description="while a task runs, sqb started with SIGINT ignored", + orders_model=ORDERS_MODEL, + environment={ + "ORDERS_TASK_SLEEP_SECONDS": "60", + "ORDERS_TASK_AFTER_SLEEP": "slept through\n", + }, + wait_sql="SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'", + expected_output=EMITTED_TAIL, + unfinished_output=b"slept through", + interrupts=signal.SIG_IGN, + program=NATIVE_SQB, + ), TerminatedCommandTestCase( description="while a long warehouse statement runs", orders_model=( @@ -138,6 +153,8 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit terminated: subprocess.CompletedProcess[bytes] = terminated_build( project_dir, wait_sql=test_case.wait_sql, + interrupts=test_case.interrupts, + program=test_case.program, SQLBUILD_RECALL_HEARTBEAT_SECONDS="0.5", **test_case.environment, ) diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index fcfcbee313..528720c34b 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -134,9 +134,11 @@ can contain sensitive SQL, output and audit evidence. A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it cancels the statements it is running, records its real end time, its output and exit code 143, -prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process -at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run -stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +prints `error: terminated by SIGTERM`, and exits with code 143. It does this also when it was +started with SIGINT ignored, as a script starts a background job and some process managers start a +service; SIGINT and Ctrl-C then stay ignored. A second SIGTERM ends the process at once. Recall +keeps heartbeating while the command discovers and compiles the project, so a run stopped in any +phase is never mistaken for one that died. An `abandoned` run cannot record its end: its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. ## Selection