diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index 6fab8e5b46..a75c92a25f 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -51,7 +51,6 @@ const ( requestTimeout = 2 * time.Minute connectTimeout = 15 * time.Second responseHeaderTimeout = 30 * time.Second - candidateTimeout = 15 * time.Second maxArchiveMembers = 64 runtimeSetLockName = ".codebase-memory-mcp-runtime.lock" @@ -1008,16 +1007,19 @@ func validateWindowsZipMember(raw string) (string, error) { } func verifyCandidate(path string) error { - ctx, cancel := context.WithTimeout(context.Background(), candidateTimeout) - defer cancel() - cmd := exec.CommandContext(ctx, path, "--version") + return verifyCandidateWithCommand(path, exec.CommandContext) +} + +func verifyCandidateWithCommand( + path string, command func(context.Context, string, ...string) *exec.Cmd, +) error { + // Verification is decided by the child exit; the smoke harness bounds hangs. + ctx := context.Background() + cmd := command(ctx, path, "--version") cmd.Stdin = nil cmd.Stdout = io.Discard cmd.Stderr = io.Discard if err := cmd.Run(); err != nil { - if ctx.Err() != nil { - return fmt.Errorf("downloaded binary verification timed out: %w", ctx.Err()) - } return fmt.Errorf("downloaded binary failed to run: %w", err) } return nil diff --git a/pkg/go/cmd/codebase-memory-mcp/main_test.go b/pkg/go/cmd/codebase-memory-mcp/main_test.go index 17a9f0e1a3..25d5fddc1b 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main_test.go +++ b/pkg/go/cmd/codebase-memory-mcp/main_test.go @@ -5,10 +5,12 @@ import ( "archive/zip" "bytes" "compress/gzip" + "context" "encoding/json" "errors" "fmt" "io" + "net" "net/http" "net/http/httptest" "os" @@ -1388,3 +1390,101 @@ func TestReleaseVersionFromModuleVersion(t *testing.T) { } } } + +// Re-executed as a candidate binary; the connection holds it until released. +func TestVerifyCandidateProcess(t *testing.T) { + address := os.Getenv("CBM_TEST_VERIFY_ADDRESS") + if address == "" { + return + } + connection, err := net.Dial("tcp", address) + if err != nil { + os.Exit(2) + } + defer connection.Close() + if _, err := connection.Write([]byte("ready")); err != nil { + os.Exit(3) + } + var release [1]byte + if _, err := io.ReadFull(connection, release[:]); err != nil { + os.Exit(4) + } + os.Exit(int(release[0])) +} + +func TestVerifyCandidateWaitsForResult(t *testing.T) { + for _, exitCode := range []byte{0, 7} { + t.Run(fmt.Sprintf("exit_%d", exitCode), func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + // This bound only aborts a hung test, never decides a successful result. + guard, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + accepted := make(chan net.Conn, 1) + go func() { + connection, err := listener.Accept() + if err == nil { + accepted <- connection + } + }() + done := make(chan error, 1) + go func() { + done <- verifyCandidateWithCommand("candidate", func(ctx context.Context, path string, args ...string) *exec.Cmd { + if _, deadline := ctx.Deadline(); deadline { + t.Error("candidate verification must wait for its result, without a deadline") + } + if path != "candidate" || !reflect.DeepEqual(args, []string{"--version"}) { + t.Errorf("candidate command = %q %q", path, args) + } + command := exec.CommandContext(guard, os.Args[0], "-test.run=^TestVerifyCandidateProcess$") + command.Env = append(os.Environ(), "CBM_TEST_VERIFY_ADDRESS="+listener.Addr().String()) + return command + }) + }() + var connection net.Conn + select { + case connection = <-accepted: + case err := <-done: + t.Fatalf("verification returned before handshake: %v", err) + case <-guard.Done(): + t.Fatal("candidate handshake hung") + } + defer connection.Close() + if err := connection.SetDeadline(time.Now().Add(time.Minute)); err != nil { + t.Fatal(err) + } + ready := make([]byte, len("ready")) + if _, err := io.ReadFull(connection, ready); err != nil || string(ready) != "ready" { + t.Fatalf("candidate handshake = %q, %v", ready, err) + } + select { + case err := <-done: + t.Fatalf("verification returned while candidate held: %v", err) + default: + } + if _, err := connection.Write([]byte{exitCode}); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if exitCode == 0 && err != nil { + t.Fatal(err) + } + if exitCode != 0 { + var exitError *exec.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != int(exitCode) { + t.Fatalf("verification error = %v, want exit %d", err, exitCode) + } + } + case <-guard.Done(): + t.Fatal("verification hung after candidate release") + } + }) + } + if err := verifyCandidate(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("verification accepted a missing executable") + } +} diff --git a/scripts/ci/test-package-wrappers.sh b/scripts/ci/test-package-wrappers.sh index 11c8d7643e..fbe004a62d 100755 --- a/scripts/ci/test-package-wrappers.sh +++ b/scripts/ci/test-package-wrappers.sh @@ -52,6 +52,8 @@ if ! "${PYTHON[@]}" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3 exit 2 fi +"${PYTHON[@]}" "$ROOT/tests/test_smoke_wrapper_run.py" + echo "=== Package wrapper: Go ===" ( cd "$ROOT/pkg/go" diff --git a/scripts/smoke-wrapper-run.py b/scripts/smoke-wrapper-run.py new file mode 100644 index 0000000000..26b3d3df4a --- /dev/null +++ b/scripts/smoke-wrapper-run.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Run a wrapper smoke command; reserve the outer bound for hang diagnosis.""" + +import os +import signal +import subprocess +import sys + + +def run(command, timeout=600): + process = subprocess.Popen(command, start_new_session=os.name != "nt") + try: + return process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + print("HANG go: wrapper smoke did not complete within the outer bound " + "(download, verification or execution); terminating its process tree", + file=sys.stderr, flush=True) + # Kill descendants too: killing only the wrapper leaves its verifier alive. + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], + check=True, timeout=30) + else: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=30) + return 124 + + +if __name__ == "__main__": + try: + result = run(sys.argv[1:]) + except (OSError, subprocess.SubprocessError) as error: + print(f"wrapper smoke could not run or clean up: {error}", file=sys.stderr) + sys.exit(125) + sys.exit(result if result >= 0 else 128 - result) diff --git a/scripts/smoke-wrappers.sh b/scripts/smoke-wrappers.sh index 300d79c90d..99a385ea26 100644 --- a/scripts/smoke-wrappers.sh +++ b/scripts/smoke-wrappers.sh @@ -140,8 +140,18 @@ smoke_go() { ran go "$dir/install.log" "$rc" return fi - # The first run downloads the binary into the wrapper's cache. - isolated "$dir" "$dir/codebase-memory-mcp$EXE" --version >"$dir/run.log" 2>&1 || rc=$? + local py="" + if command -v python3 >/dev/null 2>&1; then + py=python3 + elif command -v python >/dev/null 2>&1; then + py=python + else + missing go "Python 3 (outer hang guard)" + return + fi + # The first run downloads the binary. The outer guard diagnoses hangs; + # the verifier itself waits for the binary's exit, regardless of runner speed. + isolated "$dir" "$py" "$ROOT/scripts/smoke-wrapper-run.py" "$dir/codebase-memory-mcp$EXE" --version >"$dir/run.log" 2>&1 || rc=$? ran go "$dir/run.log" "$rc" } diff --git a/scripts/test-windows.ps1 b/scripts/test-windows.ps1 index 28739163d4..64b7723a62 100644 --- a/scripts/test-windows.ps1 +++ b/scripts/test-windows.ps1 @@ -160,6 +160,7 @@ try { # UI (build target cbm-with-ui); against a non-UI binary it reports a precondition # (exit 2) and is skipped rather than failed. $guards = @( + "tests\windows\test_worker_trace.py", "tests\windows\test_non_ascii_path.py", "tests\windows\test_non_ascii_cache_dump.py", "tests\windows\test_daemon_lifecycle.py", diff --git a/tests/test_smoke_wrapper_run.py b/tests/test_smoke_wrapper_run.py new file mode 100644 index 0000000000..e5b09b5afa --- /dev/null +++ b/tests/test_smoke_wrapper_run.py @@ -0,0 +1,99 @@ +"""The smoke watchdog reports hangs separately and cleans the command tree.""" + +import importlib.util +import pathlib +import queue +import subprocess +import sys +import threading +import unittest +from unittest import mock + +SPEC = importlib.util.spec_from_file_location( + "smoke_wrapper_run", + pathlib.Path(__file__).resolve().parents[1] / "scripts/smoke-wrapper-run.py", +) +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +class WrapperWatchdogTest(unittest.TestCase): + def test_preserves_actual_exit(self): + for code in (0, 7): + with self.subTest(code=code): + self.assertEqual(runner.run([sys.executable, "-c", f"raise SystemExit({code})"]), code) + + def test_start_failure(self): + with self.assertRaises(FileNotFoundError): + runner.run([str(pathlib.Path(__file__).resolve().parent / "missing-wrapper-2877")]) + + def test_hang_stops_real_process(self): + # Inject the outer expiry only after the child is ready, without sleeps. + real_popen = subprocess.Popen + child = None + + def start(command, **options): + nonlocal child + child = real_popen(command, stdout=subprocess.PIPE, text=True, **options) + ready = queue.Queue() + threading.Thread(target=lambda: ready.put(child.stdout.readline()), daemon=True).start() + self.assertEqual(ready.get(timeout=30), "ready\n") + real_wait = child.wait + expired = False + + def wait(timeout): + nonlocal expired + if not expired: + expired = True + raise subprocess.TimeoutExpired(command, timeout) + return real_wait(timeout=timeout) + + child.wait = wait + return child + + try: + # taskkill uses Popen too; let that separate command run normally. + def spawn(command, **options): + return real_popen(command, **options) if command[0] == "taskkill" else start(command, **options) + with mock.patch.object(runner.subprocess, "Popen", side_effect=spawn), \ + mock.patch.object(runner.sys, "stderr"): + self.assertEqual(runner.run([ + sys.executable, "-u", "-c", + "import sys, threading; sys.stdout.buffer.write(b'ready\\r\\n'); " + "sys.stdout.buffer.flush(); threading.Event().wait()", + ]), 124) + self.assertIsNotNone(child.poll()) + finally: + if child is not None: + if child.poll() is None: + child.kill() + child.wait(timeout=30) + child.stdout.close() + + def test_hang_kills_tree_and_reaps(self): + for platform in ("posix", "nt"): + with self.subTest(platform=platform), \ + mock.patch.object(runner.os, "name", platform), \ + mock.patch.object(runner.subprocess, "Popen") as popen, \ + mock.patch.object(runner.subprocess, "run") as taskkill, \ + mock.patch.object(runner.os, "killpg", create=True) as killpg, \ + mock.patch.object(runner.signal, "SIGKILL", 9, create=True), \ + mock.patch.object(runner.sys, "stderr") as diagnostic: + process = popen.return_value + process.pid = 123 + process.wait.side_effect = [subprocess.TimeoutExpired("candidate", 600), 0] + self.assertEqual(runner.run(["candidate"]), 124) + popen.assert_called_once_with(["candidate"], start_new_session=platform != "nt") + self.assertEqual(process.wait.call_args_list, [mock.call(timeout=600), mock.call(timeout=30)]) + if platform == "nt": + taskkill.assert_called_once_with( + ["taskkill", "/PID", "123", "/T", "/F"], check=True, timeout=30) + killpg.assert_not_called() + else: + killpg.assert_called_once_with(123, runner.signal.SIGKILL) + taskkill.assert_not_called() + self.assertIn("HANG go", "".join(call.args[0] for call in diagnostic.write.call_args_list)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/windows/test_daemon_stability.py b/tests/windows/test_daemon_stability.py index 962628d6c9..3a5b526f8d 100644 --- a/tests/windows/test_daemon_stability.py +++ b/tests/windows/test_daemon_stability.py @@ -520,6 +520,29 @@ def section_churn_stability(binary, work): kill_pid(daemon_pid) +def read_cleanup_trace(err_path, cache, traces): + # The supervisor removes successful worker logs while this guard polls them. + paths = [err_path] + log_dir = os.path.join(cache, "logs") + if os.path.isdir(log_dir): + paths.extend(os.path.join(log_dir, name) for name in os.listdir(log_dir) + if name.startswith(".worker-log-")) + unavailable = [] + for path in paths: + try: + with open(path, "rb") as handle: + traces[path] = handle.read() + except FileNotFoundError: + pass + except PermissionError as error: + # Windows also refuses opens during deletion/sharing contention. + # Keep the last trace and retry on the next poll; retain diagnostics. + if path == err_path or getattr(error, "winerror", None) not in (5, 32): + raise + unavailable.append(("worker trace temporarily unreadable: " + str(error)).encode()) + return b"\n".join(list(traces.values()) + unavailable) + + def section_local_cleanup_waits_for_startup_gate(binary, work): """A live startup owner must not turn completed local work into failure.""" if os.name != "nt": @@ -553,22 +576,6 @@ def section_local_cleanup_waits_for_startup_gate(binary, work): locked = False traces = {} - def read_trace(): - # The daemon supervises indexing in a worker. Worker progress goes to - # its own durable log; the outer CLI only reports the final response. - paths = [err_path] - log_dir = os.path.join(cache, "logs") - if os.path.isdir(log_dir): - paths.extend(os.path.join(log_dir, name) for name in os.listdir(log_dir) - if name.startswith(".worker-log-")) - for path in paths: - try: - with open(path, "rb") as handle: - traces[path] = handle.read() - except FileNotFoundError: - pass # The supervisor removes a successful worker's log. - return b"\n".join(traces.values()) - with open(gates[0], "r+b") as gate, open(err_path, "wb") as errors: try: process = subprocess.Popen( @@ -581,7 +588,7 @@ def read_trace(): # Extraction is after begin_work released startup-v2. Observe that # event before acquiring the real one-byte native startup lock. while time.monotonic() < deadline: - trace = read_trace() + trace = read_cleanup_trace(err_path, cache, traces) if b"msg=parallel.extract" in trace or b"msg=pass.definitions" in trace: gate.seek(0) try: @@ -598,7 +605,7 @@ def read_trace(): if not locked: raise SetupFailure("index never entered local work:\n" + excerpt(trace)) while time.monotonic() < deadline: - trace = read_trace() + trace = read_cleanup_trace(err_path, cache, traces) if b"Completed index_repository (" in trace: break if process.poll() is not None: @@ -619,7 +626,7 @@ def read_trace(): msvcrt.locking(gate.fileno(), msvcrt.LK_UNLCK, 1) locked = False stdout, _ = process.communicate(timeout=30) - trace = read_trace() + trace = read_cleanup_trace(err_path, cache, traces) if not stayed_alive or process.returncode != 0 or b"cleanup_timeout" in trace: print("RED: completed work must wait out startup contention and exit 0 " "after release (rc=%s):\n%s" % diff --git a/tests/windows/test_worker_trace.py b/tests/windows/test_worker_trace.py new file mode 100644 index 0000000000..f9b5103a41 --- /dev/null +++ b/tests/windows/test_worker_trace.py @@ -0,0 +1,66 @@ +"""Worker trace polling tolerates Windows cleanup races without hiding I/O errors.""" +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +from test_daemon_stability import read_cleanup_trace + + +class WorkerTraceTest(unittest.TestCase): + def test_worker_cleanup_race_preserves_trace_and_retries(self): + for winerror in (5, 32): + with self.subTest(winerror=winerror), tempfile.TemporaryDirectory() as cache: + Path(cache, "logs").mkdir() + worker = str(Path(cache, "logs", ".worker-log-fixture")) + stderr = str(Path(cache, "stderr")) + Path(stderr).write_bytes(b"cli") + Path(worker).write_bytes(b"previous") + traces = {} + self.assertIn(b"previous", read_cleanup_trace(stderr, cache, traces)) + denied = PermissionError(13, "Permission denied", worker) + denied.winerror = winerror + real_open = open + + def read(path, *args, **kwargs): + if path == worker: + raise denied + return real_open(path, *args, **kwargs) + + with mock.patch("builtins.open", side_effect=read): + trace = read_cleanup_trace(stderr, cache, traces) + self.assertIn(b"previous", trace) + self.assertIn(b"temporarily unreadable", trace) + Path(worker).write_bytes(b"current") + self.assertIn(b"current", read_cleanup_trace(stderr, cache, traces)) + os.unlink(worker) + self.assertIn(b"current", read_cleanup_trace(stderr, cache, traces)) + + def test_stderr_and_other_permission_errors_propagate(self): + with tempfile.TemporaryDirectory() as cache: + stderr = str(Path(cache, "stderr")) + denied = PermissionError(13, "Permission denied", stderr) + denied.winerror = 5 + with mock.patch("builtins.open", side_effect=denied): + with self.assertRaises(PermissionError): + read_cleanup_trace(stderr, cache, {}) + Path(stderr).write_bytes(b"cli") + Path(cache, "logs").mkdir() + worker = str(Path(cache, "logs", ".worker-log-fixture")) + Path(worker).write_bytes(b"worker") + real_open = open + + def read(path, *args, **kwargs): + if path == worker: + raise PermissionError(13, "Permission denied", worker) + return real_open(path, *args, **kwargs) + + with mock.patch("builtins.open", side_effect=read): + with self.assertRaises(PermissionError): + read_cleanup_trace(stderr, cache, {}) + + +if __name__ == "__main__": + unittest.main(argv=[sys.argv[0]])