Skip to content

Commit c10b78c

Browse files
committed
test: fail a hung E2E command with every thread's stack instead of stalling the job
run_sqb now gives each command ten minutes. A command still running then has its process group aborted with Python's fault handler enabled, so the failing test reports every thread's stack, and the group is killed. Standard input ends at once when a test passes none.
1 parent c9d31bd commit c10b78c

4 files changed

Lines changed: 193 additions & 7 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,8 @@ E2E_DUCKDB_CLI_PATHS := \
179179
tests/e2e/src/sqlbuild/cli/commands/main/snowflake \
180180
tests/e2e/src/sqlbuild/cli/commands/main/sqlite \
181181
tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \
182-
tests/e2e/src/sqlbuild/cli/commands/main/test
182+
tests/e2e/src/sqlbuild/cli/commands/main/test \
183+
tests/e2e/src/sqlbuild/cli/commands/shared
183184

184185
E2E_DUCKDB_INSPECTION_PATHS := \
185186
tests/e2e/src/sqlbuild/cli/commands/main/clone \
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
"""Test case types for the shared E2E command helpers."""
2+
3+
from __future__ import annotations
4+
5+
from dataclasses import dataclass
6+
7+
8+
@dataclass(frozen=True)
9+
class HangDumpTestCase:
10+
"""A Python command that never finishes and what its hang report must show."""
11+
12+
description: str
13+
script: str
14+
expected_fragments: tuple[str, ...]
15+
16+
17+
@dataclass(frozen=True)
18+
class FinishedCommandTestCase:
19+
"""A Python command that finishes and the result the helper must return for it."""
20+
21+
description: str
22+
script: str
23+
input_text: str | None
24+
expected_result: tuple[int, str, str]

‎tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py‎

Lines changed: 72 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import os
77
import pty
88
import re
9+
import shlex
10+
import signal
911
import struct
1012
import subprocess
1113
import termios
@@ -22,6 +24,10 @@
2224
SOURCE_LOADER_STRATEGIES_DIR: Path = (
2325
REPO_ROOT / "tests" / "e2e" / "fixtures" / "source_loader_strategies"
2426
)
27+
# Far above any healthy command, so only a hang trips it: one test fails instead of the CI job.
28+
SQB_COMMAND_TIMEOUT_SECONDS: float = 600.0
29+
_HANG_DUMP_GRACE_SECONDS: float = 30.0
30+
_HANG_OUTPUT_TAIL_CHARACTERS: int = 20_000
2531

2632

2733
def prepare_waffle_shop(tmp_path: Path) -> Path:
@@ -208,18 +214,78 @@ def run_sqb(
208214

209215
process_env: dict[str, str] = dict(os.environ)
210216
process_env.update(env or {})
217+
process_env.setdefault("PYTHONFAULTHANDLER", "1")
211218

212-
return subprocess.run(
213-
["uv", "run", "sqb", "--project-dir", str(project_dir), *command],
219+
return run_with_hang_dump(
220+
argv=("uv", "run", "sqb", "--project-dir", str(project_dir), *command),
214221
cwd=working_dir or REPO_ROOT,
215-
capture_output=True,
216-
text=True,
217-
input=input_text,
218222
env=process_env,
219-
check=False,
223+
input_text=input_text,
224+
timeout_seconds=SQB_COMMAND_TIMEOUT_SECONDS,
225+
)
226+
227+
228+
def run_with_hang_dump(
229+
*,
230+
argv: tuple[str, ...],
231+
cwd: Path,
232+
env: Mapping[str, str],
233+
input_text: str | None,
234+
timeout_seconds: float,
235+
) -> subprocess.CompletedProcess[str]:
236+
"""Run `argv` to completion, or fail with every Python thread's stack once it hangs.
237+
238+
Standard input carries `input_text`, or ends at once without it. The command runs in its own
239+
process group. Past `timeout_seconds` the whole group receives
240+
SIGABRT, so Python processes started with `PYTHONFAULTHANDLER` print all thread stacks to
241+
stderr before they exit, and the group is then killed.
242+
"""
243+
244+
process: subprocess.Popen[str] = subprocess.Popen(
245+
argv,
246+
cwd=cwd,
247+
stdin=subprocess.PIPE,
248+
stdout=subprocess.PIPE,
249+
stderr=subprocess.PIPE,
250+
text=True,
251+
env=dict(env),
252+
process_group=0,
253+
)
254+
try:
255+
stdout, stderr = process.communicate(input=input_text, timeout=timeout_seconds)
256+
except subprocess.TimeoutExpired:
257+
stdout, stderr = _dump_and_kill_group(process=process)
258+
raise AssertionError(
259+
f"{shlex.join(argv)} did not finish within {timeout_seconds:.0f}s; "
260+
"its process group was aborted to dump thread stacks.\n"
261+
f"--- stdout (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n"
262+
f"{stdout[-_HANG_OUTPUT_TAIL_CHARACTERS:]}\n"
263+
f"--- stderr (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n"
264+
f"{stderr[-_HANG_OUTPUT_TAIL_CHARACTERS:]}"
265+
) from None
266+
return subprocess.CompletedProcess(
267+
args=list(argv), returncode=process.returncode, stdout=stdout, stderr=stderr
220268
)
221269

222270

271+
def _dump_and_kill_group(*, process: subprocess.Popen[str]) -> tuple[str, str]:
272+
_signal_group(process_group=process.pid, signal_number=signal.SIGABRT)
273+
try:
274+
stdout, stderr = process.communicate(timeout=_HANG_DUMP_GRACE_SECONDS)
275+
except subprocess.TimeoutExpired:
276+
_signal_group(process_group=process.pid, signal_number=signal.SIGKILL)
277+
stdout, stderr = process.communicate()
278+
_signal_group(process_group=process.pid, signal_number=signal.SIGKILL)
279+
return stdout or "", stderr or ""
280+
281+
282+
def _signal_group(*, process_group: int, signal_number: signal.Signals) -> None:
283+
try:
284+
os.killpg(process_group, signal_number)
285+
except ProcessLookupError:
286+
return
287+
288+
223289
def run_sqb_with_pty(
224290
*,
225291
command: tuple[str, ...],
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
"""A hung command fails its own test with every Python thread's stack instead of hanging the run."""
2+
3+
from __future__ import annotations
4+
5+
import os
6+
import subprocess
7+
import sys
8+
import time
9+
from pathlib import Path
10+
11+
import pytest
12+
13+
from tests.e2e.src.sqlbuild.cli.commands.shared._test_types import (
14+
FinishedCommandTestCase,
15+
HangDumpTestCase,
16+
)
17+
from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_with_hang_dump
18+
19+
20+
@pytest.mark.parametrize(
21+
"test_case",
22+
[
23+
HangDumpTestCase(
24+
description="main thread joins a worker blocked forever",
25+
script=(
26+
"import threading\n"
27+
"def blocked_worker_wait():\n"
28+
" threading.Event().wait()\n"
29+
"def joining_main_thread():\n"
30+
" worker = threading.Thread(target=blocked_worker_wait)\n"
31+
" worker.start()\n"
32+
" print('started', flush=True)\n"
33+
" worker.join()\n"
34+
"joining_main_thread()\n"
35+
),
36+
expected_fragments=("started", "blocked_worker_wait", "joining_main_thread"),
37+
),
38+
],
39+
ids=lambda case: case.description,
40+
)
41+
def test_given_hung_python_command_when_running_with_hang_dump_then_fails_with_thread_stacks(
42+
test_case: HangDumpTestCase, tmp_path: Path
43+
) -> None:
44+
started: float = time.monotonic()
45+
with pytest.raises(AssertionError) as raised:
46+
run_with_hang_dump(
47+
argv=(sys.executable, "-c", test_case.script),
48+
cwd=tmp_path,
49+
env={**os.environ, "PYTHONFAULTHANDLER": "1"},
50+
input_text=None,
51+
timeout_seconds=2.0,
52+
)
53+
54+
message: str = str(raised.value)
55+
assert "did not finish within 2s" in message
56+
assert [fragment in message for fragment in test_case.expected_fragments] == [True] * len(
57+
test_case.expected_fragments
58+
), message
59+
assert time.monotonic() - started < 30.0
60+
61+
62+
@pytest.mark.parametrize(
63+
"test_case",
64+
[
65+
FinishedCommandTestCase(
66+
description="input reaches the command and its exit code and output return",
67+
script="import sys; sys.stdout.write(sys.stdin.read().upper()); sys.exit(3)",
68+
input_text="orders\n",
69+
expected_result=(3, "ORDERS\n", ""),
70+
),
71+
FinishedCommandTestCase(
72+
description="without input the command reads end of input",
73+
script="import sys; print(repr(sys.stdin.read()))",
74+
input_text=None,
75+
expected_result=(0, "''\n", ""),
76+
),
77+
],
78+
ids=lambda case: case.description,
79+
)
80+
def test_given_finishing_command_when_running_with_hang_dump_then_returns_its_result(
81+
test_case: FinishedCommandTestCase, tmp_path: Path
82+
) -> None:
83+
completed: subprocess.CompletedProcess[str] = run_with_hang_dump(
84+
argv=(sys.executable, "-c", test_case.script),
85+
cwd=tmp_path,
86+
env=dict(os.environ),
87+
input_text=test_case.input_text,
88+
timeout_seconds=60.0,
89+
)
90+
91+
assert (completed.returncode, completed.stdout, completed.stderr) == test_case.expected_result
92+
93+
94+
if __name__ == "__main__":
95+
pytest.main([__file__, "-vv"])

0 commit comments

Comments
 (0)