|
6 | 6 | import os |
7 | 7 | import pty |
8 | 8 | import re |
| 9 | +import shlex |
| 10 | +import signal |
9 | 11 | import struct |
10 | 12 | import subprocess |
11 | 13 | import termios |
|
22 | 24 | SOURCE_LOADER_STRATEGIES_DIR: Path = ( |
23 | 25 | REPO_ROOT / "tests" / "e2e" / "fixtures" / "source_loader_strategies" |
24 | 26 | ) |
| 27 | +# Far above any healthy command, so only a hang trips it: one test fails instead of the CI job. |
| 28 | +SQB_COMMAND_TIMEOUT_SECONDS: float = 600.0 |
| 29 | +_HANG_DUMP_GRACE_SECONDS: float = 30.0 |
| 30 | +_HANG_OUTPUT_TAIL_CHARACTERS: int = 20_000 |
25 | 31 |
|
26 | 32 |
|
27 | 33 | def prepare_waffle_shop(tmp_path: Path) -> Path: |
@@ -208,18 +214,78 @@ def run_sqb( |
208 | 214 |
|
209 | 215 | process_env: dict[str, str] = dict(os.environ) |
210 | 216 | process_env.update(env or {}) |
| 217 | + process_env.setdefault("PYTHONFAULTHANDLER", "1") |
211 | 218 |
|
212 | | - return subprocess.run( |
213 | | - ["uv", "run", "sqb", "--project-dir", str(project_dir), *command], |
| 219 | + return run_with_hang_dump( |
| 220 | + argv=("uv", "run", "sqb", "--project-dir", str(project_dir), *command), |
214 | 221 | cwd=working_dir or REPO_ROOT, |
215 | | - capture_output=True, |
216 | | - text=True, |
217 | | - input=input_text, |
218 | 222 | env=process_env, |
219 | | - check=False, |
| 223 | + input_text=input_text, |
| 224 | + timeout_seconds=SQB_COMMAND_TIMEOUT_SECONDS, |
| 225 | + ) |
| 226 | + |
| 227 | + |
| 228 | +def run_with_hang_dump( |
| 229 | + *, |
| 230 | + argv: tuple[str, ...], |
| 231 | + cwd: Path, |
| 232 | + env: Mapping[str, str], |
| 233 | + input_text: str | None, |
| 234 | + timeout_seconds: float, |
| 235 | +) -> subprocess.CompletedProcess[str]: |
| 236 | + """Run `argv` to completion, or fail with every Python thread's stack once it hangs. |
| 237 | +
|
| 238 | + Standard input carries `input_text`, or ends at once without it. The command runs in its own |
| 239 | + process group. Past `timeout_seconds` the whole group receives |
| 240 | + SIGABRT, so Python processes started with `PYTHONFAULTHANDLER` print all thread stacks to |
| 241 | + stderr before they exit, and the group is then killed. |
| 242 | + """ |
| 243 | + |
| 244 | + process: subprocess.Popen[str] = subprocess.Popen( |
| 245 | + argv, |
| 246 | + cwd=cwd, |
| 247 | + stdin=subprocess.PIPE, |
| 248 | + stdout=subprocess.PIPE, |
| 249 | + stderr=subprocess.PIPE, |
| 250 | + text=True, |
| 251 | + env=dict(env), |
| 252 | + process_group=0, |
| 253 | + ) |
| 254 | + try: |
| 255 | + stdout, stderr = process.communicate(input=input_text, timeout=timeout_seconds) |
| 256 | + except subprocess.TimeoutExpired: |
| 257 | + stdout, stderr = _dump_and_kill_group(process=process) |
| 258 | + raise AssertionError( |
| 259 | + f"{shlex.join(argv)} did not finish within {timeout_seconds:.0f}s; " |
| 260 | + "its process group was aborted to dump thread stacks.\n" |
| 261 | + f"--- stdout (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n" |
| 262 | + f"{stdout[-_HANG_OUTPUT_TAIL_CHARACTERS:]}\n" |
| 263 | + f"--- stderr (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n" |
| 264 | + f"{stderr[-_HANG_OUTPUT_TAIL_CHARACTERS:]}" |
| 265 | + ) from None |
| 266 | + return subprocess.CompletedProcess( |
| 267 | + args=list(argv), returncode=process.returncode, stdout=stdout, stderr=stderr |
220 | 268 | ) |
221 | 269 |
|
222 | 270 |
|
| 271 | +def _dump_and_kill_group(*, process: subprocess.Popen[str]) -> tuple[str, str]: |
| 272 | + _signal_group(process_group=process.pid, signal_number=signal.SIGABRT) |
| 273 | + try: |
| 274 | + stdout, stderr = process.communicate(timeout=_HANG_DUMP_GRACE_SECONDS) |
| 275 | + except subprocess.TimeoutExpired: |
| 276 | + _signal_group(process_group=process.pid, signal_number=signal.SIGKILL) |
| 277 | + stdout, stderr = process.communicate() |
| 278 | + _signal_group(process_group=process.pid, signal_number=signal.SIGKILL) |
| 279 | + return stdout or "", stderr or "" |
| 280 | + |
| 281 | + |
| 282 | +def _signal_group(*, process_group: int, signal_number: signal.Signals) -> None: |
| 283 | + try: |
| 284 | + os.killpg(process_group, signal_number) |
| 285 | + except ProcessLookupError: |
| 286 | + return |
| 287 | + |
| 288 | + |
223 | 289 | def run_sqb_with_pty( |
224 | 290 | *, |
225 | 291 | command: tuple[str, ...], |
|
0 commit comments