Skip to content

Commit 21727f6

Browse files
authored
fix: stop compute log capture deadlocking a command at exit (#1165)
* fix: release the compute log lock before rejecting a write to a finished capture A write that reached a capture already marked complete built its rejection while still holding the storage lock. Building it imports Python, which can release the GIL; the main thread then pruning the logs waited for that lock while holding the GIL, so neither could continue and the command never exited. The lock is now released before the rejection is raised. * test: fail a hung E2E command with every thread's stack instead of stalling the job run_sqb now gives each command ten minutes. A command still running then has its process group aborted with Python's fault handler enabled, so the failing test reports every thread's stack, and the group is killed. Standard input ends at once when a test passes none.
1 parent ad47322 commit 21727f6

8 files changed

Lines changed: 339 additions & 7 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,8 @@ E2E_DUCKDB_CLI_PATHS := \
179179
tests/e2e/src/sqlbuild/cli/commands/main/snowflake \
180180
tests/e2e/src/sqlbuild/cli/commands/main/sqlite \
181181
tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \
182-
tests/e2e/src/sqlbuild/cli/commands/main/test
182+
tests/e2e/src/sqlbuild/cli/commands/main/test \
183+
tests/e2e/src/sqlbuild/cli/commands/shared
183184

184185
E2E_DUCKDB_INSPECTION_PATHS := \
185186
tests/e2e/src/sqlbuild/cli/commands/main/clone \

‎crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ fn exact_count(value: &Bound<'_, PyAny>, fallback: i64) -> i64 {
9090
}
9191

9292
impl LocalFilesystemComputeLogStorage {
93+
/// Callers wait for this lock holding the GIL, so never run Python while holding it.
9394
fn locked(&self) -> PyResult<MutexGuard<'_, StorageState>> {
9495
self.state
9596
.lock()
@@ -116,6 +117,7 @@ impl LocalFilesystemComputeLogStorage {
116117
) -> PyResult<()> {
117118
let mut state = self.locked()?;
118119
let Some(writers) = state.writers.get_mut(invocation_id) else {
120+
drop(state);
119121
return Err(not_active(py));
120122
};
121123
writers[stream].write_all(data)?;
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
"""Test case types for the shared E2E command helpers."""
2+
3+
from __future__ import annotations
4+
5+
from dataclasses import dataclass
6+
7+
8+
@dataclass(frozen=True)
9+
class HangDumpTestCase:
10+
"""A Python command that never finishes and what its hang report must show."""
11+
12+
description: str
13+
script: str
14+
expected_fragments: tuple[str, ...]
15+
16+
17+
@dataclass(frozen=True)
18+
class FinishedCommandTestCase:
19+
"""A Python command that finishes and the result the helper must return for it."""
20+
21+
description: str
22+
script: str
23+
input_text: str | None
24+
expected_result: tuple[int, str, str]

‎tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py‎

Lines changed: 72 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import os
77
import pty
88
import re
9+
import shlex
10+
import signal
911
import struct
1012
import subprocess
1113
import termios
@@ -22,6 +24,10 @@
2224
SOURCE_LOADER_STRATEGIES_DIR: Path = (
2325
REPO_ROOT / "tests" / "e2e" / "fixtures" / "source_loader_strategies"
2426
)
27+
# Far above any healthy command, so only a hang trips it: one test fails instead of the CI job.
28+
SQB_COMMAND_TIMEOUT_SECONDS: float = 600.0
29+
_HANG_DUMP_GRACE_SECONDS: float = 30.0
30+
_HANG_OUTPUT_TAIL_CHARACTERS: int = 20_000
2531

2632

2733
def prepare_waffle_shop(tmp_path: Path) -> Path:
@@ -208,18 +214,78 @@ def run_sqb(
208214

209215
process_env: dict[str, str] = dict(os.environ)
210216
process_env.update(env or {})
217+
process_env.setdefault("PYTHONFAULTHANDLER", "1")
211218

212-
return subprocess.run(
213-
["uv", "run", "sqb", "--project-dir", str(project_dir), *command],
219+
return run_with_hang_dump(
220+
argv=("uv", "run", "sqb", "--project-dir", str(project_dir), *command),
214221
cwd=working_dir or REPO_ROOT,
215-
capture_output=True,
216-
text=True,
217-
input=input_text,
218222
env=process_env,
219-
check=False,
223+
input_text=input_text,
224+
timeout_seconds=SQB_COMMAND_TIMEOUT_SECONDS,
225+
)
226+
227+
228+
def run_with_hang_dump(
229+
*,
230+
argv: tuple[str, ...],
231+
cwd: Path,
232+
env: Mapping[str, str],
233+
input_text: str | None,
234+
timeout_seconds: float,
235+
) -> subprocess.CompletedProcess[str]:
236+
"""Run `argv` to completion, or fail with every Python thread's stack once it hangs.
237+
238+
Standard input carries `input_text`, or ends at once without it. The command runs in its own
239+
process group. Past `timeout_seconds` the whole group receives
240+
SIGABRT, so Python processes started with `PYTHONFAULTHANDLER` print all thread stacks to
241+
stderr before they exit, and the group is then killed.
242+
"""
243+
244+
process: subprocess.Popen[str] = subprocess.Popen(
245+
argv,
246+
cwd=cwd,
247+
stdin=subprocess.PIPE,
248+
stdout=subprocess.PIPE,
249+
stderr=subprocess.PIPE,
250+
text=True,
251+
env=dict(env),
252+
process_group=0,
253+
)
254+
try:
255+
stdout, stderr = process.communicate(input=input_text, timeout=timeout_seconds)
256+
except subprocess.TimeoutExpired:
257+
stdout, stderr = _dump_and_kill_group(process=process)
258+
raise AssertionError(
259+
f"{shlex.join(argv)} did not finish within {timeout_seconds:.0f}s; "
260+
"its process group was aborted to dump thread stacks.\n"
261+
f"--- stdout (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n"
262+
f"{stdout[-_HANG_OUTPUT_TAIL_CHARACTERS:]}\n"
263+
f"--- stderr (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n"
264+
f"{stderr[-_HANG_OUTPUT_TAIL_CHARACTERS:]}"
265+
) from None
266+
return subprocess.CompletedProcess(
267+
args=list(argv), returncode=process.returncode, stdout=stdout, stderr=stderr
220268
)
221269

222270

271+
def _dump_and_kill_group(*, process: subprocess.Popen[str]) -> tuple[str, str]:
272+
_signal_group(process_group=process.pid, signal_number=signal.SIGABRT)
273+
try:
274+
stdout, stderr = process.communicate(timeout=_HANG_DUMP_GRACE_SECONDS)
275+
except subprocess.TimeoutExpired:
276+
_signal_group(process_group=process.pid, signal_number=signal.SIGKILL)
277+
stdout, stderr = process.communicate()
278+
_signal_group(process_group=process.pid, signal_number=signal.SIGKILL)
279+
return stdout or "", stderr or ""
280+
281+
282+
def _signal_group(*, process_group: int, signal_number: signal.Signals) -> None:
283+
try:
284+
os.killpg(process_group, signal_number)
285+
except ProcessLookupError:
286+
return
287+
288+
223289
def run_sqb_with_pty(
224290
*,
225291
command: tuple[str, ...],
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
"""A hung command fails its own test with every Python thread's stack instead of hanging the run."""
2+
3+
from __future__ import annotations
4+
5+
import os
6+
import subprocess
7+
import sys
8+
import time
9+
from pathlib import Path
10+
11+
import pytest
12+
13+
from tests.e2e.src.sqlbuild.cli.commands.shared._test_types import (
14+
FinishedCommandTestCase,
15+
HangDumpTestCase,
16+
)
17+
from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_with_hang_dump
18+
19+
20+
@pytest.mark.parametrize(
21+
"test_case",
22+
[
23+
HangDumpTestCase(
24+
description="main thread joins a worker blocked forever",
25+
script=(
26+
"import threading\n"
27+
"def blocked_worker_wait():\n"
28+
" threading.Event().wait()\n"
29+
"def joining_main_thread():\n"
30+
" worker = threading.Thread(target=blocked_worker_wait)\n"
31+
" worker.start()\n"
32+
" print('started', flush=True)\n"
33+
" worker.join()\n"
34+
"joining_main_thread()\n"
35+
),
36+
expected_fragments=("started", "blocked_worker_wait", "joining_main_thread"),
37+
),
38+
],
39+
ids=lambda case: case.description,
40+
)
41+
def test_given_hung_python_command_when_running_with_hang_dump_then_fails_with_thread_stacks(
42+
test_case: HangDumpTestCase, tmp_path: Path
43+
) -> None:
44+
started: float = time.monotonic()
45+
with pytest.raises(AssertionError) as raised:
46+
run_with_hang_dump(
47+
argv=(sys.executable, "-c", test_case.script),
48+
cwd=tmp_path,
49+
env={**os.environ, "PYTHONFAULTHANDLER": "1"},
50+
input_text=None,
51+
timeout_seconds=2.0,
52+
)
53+
54+
message: str = str(raised.value)
55+
assert "did not finish within 2s" in message
56+
assert [fragment in message for fragment in test_case.expected_fragments] == [True] * len(
57+
test_case.expected_fragments
58+
), message
59+
assert time.monotonic() - started < 30.0
60+
61+
62+
@pytest.mark.parametrize(
63+
"test_case",
64+
[
65+
FinishedCommandTestCase(
66+
description="input reaches the command and its exit code and output return",
67+
script="import sys; sys.stdout.write(sys.stdin.read().upper()); sys.exit(3)",
68+
input_text="orders\n",
69+
expected_result=(3, "ORDERS\n", ""),
70+
),
71+
FinishedCommandTestCase(
72+
description="without input the command reads end of input",
73+
script="import sys; print(repr(sys.stdin.read()))",
74+
input_text=None,
75+
expected_result=(0, "''\n", ""),
76+
),
77+
],
78+
ids=lambda case: case.description,
79+
)
80+
def test_given_finishing_command_when_running_with_hang_dump_then_returns_its_result(
81+
test_case: FinishedCommandTestCase, tmp_path: Path
82+
) -> None:
83+
completed: subprocess.CompletedProcess[str] = run_with_hang_dump(
84+
argv=(sys.executable, "-c", test_case.script),
85+
cwd=tmp_path,
86+
env=dict(os.environ),
87+
input_text=test_case.input_text,
88+
timeout_seconds=60.0,
89+
)
90+
91+
assert (completed.returncode, completed.stdout, completed.stderr) == test_case.expected_result
92+
93+
94+
if __name__ == "__main__":
95+
pytest.main([__file__, "-vv"])

‎tests/integration/src/sqlbuild/runtime/compute_logs/__init__.py‎

Whitespace-only changes.
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"""Test case types for compute log storage integration tests."""
2+
3+
from __future__ import annotations
4+
5+
from dataclasses import dataclass
6+
7+
8+
@dataclass(frozen=True)
9+
class StorageLockTestCase:
10+
"""A Python program making concurrent storage calls from two threads, and what it prints."""
11+
12+
description: str
13+
script: str
14+
expected_stdout: str
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
"""Compute log storage never runs Python while holding its lock, so it cannot deadlock on the GIL."""
2+
3+
from __future__ import annotations
4+
5+
import subprocess
6+
import sys
7+
from pathlib import Path
8+
from textwrap import dedent
9+
10+
import pytest
11+
12+
from tests.integration.src.sqlbuild.runtime.compute_logs._test_types import (
13+
StorageLockTestCase,
14+
)
15+
16+
_DEADLOCK_TIMEOUT_SECONDS: int = 30
17+
18+
# A late write from a background thread reaches a capture that was already marked complete.
19+
# Raising its rejection imports the storage exceptions, here forced to load afresh, and the
20+
# import's file system calls release the GIL. The finder below holds that import open until the main thread is
21+
# pruning, which takes the storage lock while holding the GIL.
22+
_LATE_APPEND_DURING_PRUNE: str = dedent(
23+
"""
24+
import dataclasses
25+
import sys
26+
import threading
27+
import time
28+
from datetime import UTC, datetime
29+
from pathlib import Path
30+
31+
from sqlbuild.compute_logs import (
32+
COMPUTE_LOG_FORMAT_VERSION,
33+
CaptureMetadata,
34+
ComputeLogStream,
35+
FinalCaptureMetadata,
36+
LocalFilesystemComputeLogStorage,
37+
)
38+
39+
EXCEPTIONS_MODULE = "sqlbuild.runtime.compute_logs.exceptions"
40+
importing = threading.Event()
41+
42+
43+
class SlowExceptionsImport:
44+
def find_spec(self, name, path=None, target=None):
45+
if name == EXCEPTIONS_MODULE and not importing.is_set():
46+
importing.set()
47+
time.sleep(0.5)
48+
return None
49+
50+
51+
project_dir = Path(sys.argv[1])
52+
started_at = datetime.now(UTC)
53+
metadata = CaptureMetadata(
54+
format_version=COMPUTE_LOG_FORMAT_VERSION,
55+
invocation_id="orders0000000000000000000000000a",
56+
command="compile",
57+
project_dir=str(project_dir),
58+
started_at=started_at,
59+
capture_date=started_at.date().isoformat(),
60+
)
61+
storage = LocalFilesystemComputeLogStorage(project_dir=project_dir)
62+
storage.start_capture(metadata)
63+
storage.mark_complete(
64+
FinalCaptureMetadata(
65+
**dataclasses.asdict(metadata),
66+
completed_at=datetime.now(UTC),
67+
exit_code=1,
68+
stdout_bytes=0,
69+
stderr_bytes=0,
70+
diagnostics_bytes=0,
71+
)
72+
)
73+
rejected = []
74+
del sys.modules[EXCEPTIONS_MODULE]
75+
sys.meta_path.insert(0, SlowExceptionsImport())
76+
77+
78+
def late_append():
79+
try:
80+
storage.append(
81+
invocation_id=metadata.invocation_id,
82+
stream=ComputeLogStream.STDERR,
83+
data=b"late customer export notice\\n",
84+
)
85+
except Exception as error:
86+
rejected.append(type(error).__name__)
87+
88+
89+
writer = threading.Thread(target=late_append)
90+
writer.start()
91+
importing.wait()
92+
storage.prune()
93+
writer.join()
94+
print(rejected)
95+
"""
96+
)
97+
98+
99+
@pytest.mark.parametrize(
100+
"test_case",
101+
(
102+
StorageLockTestCase(
103+
description="late append rejected while the main thread prunes",
104+
script=_LATE_APPEND_DURING_PRUNE,
105+
expected_stdout="['CaptureStateError']\n",
106+
),
107+
),
108+
ids=lambda case: case.description,
109+
)
110+
def test_given_rejected_append_releasing_gil_when_pruning_then_both_threads_finish(
111+
test_case: StorageLockTestCase, tmp_path: Path
112+
) -> None:
113+
try:
114+
completed: subprocess.CompletedProcess[str] = subprocess.run(
115+
[sys.executable, "-c", test_case.script, str(tmp_path)],
116+
capture_output=True,
117+
text=True,
118+
timeout=_DEADLOCK_TIMEOUT_SECONDS,
119+
check=False,
120+
)
121+
except subprocess.TimeoutExpired:
122+
pytest.fail(f"storage calls deadlocked for {_DEADLOCK_TIMEOUT_SECONDS}s")
123+
124+
assert (completed.returncode, completed.stdout) == (0, test_case.expected_stdout), (
125+
completed.stderr
126+
)
127+
128+
129+
if __name__ == "__main__":
130+
pytest.main([__file__, "-vv"])

0 commit comments

Comments
 (0)