From b9fddbe4924820e2d6ec8078ce2d8b6bfedaece1 Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 14:21:44 +0800 Subject: [PATCH 1/6] fix(host): isolate launch authority and bound native stdin dispatch Rebuild PR #6884 from its own B3 changes on main 5853e151. Preserve the current Goal authority check at native launch and request dispatch. Bound stdin delivery while holding the writer guard, count delivery against the original request deadline, and cancel and reap the original process on timeout so Stop cannot be held indefinitely by pipe pressure. Keep the existing complexity limits and preserve Unicode input. Original source: b813ae0064bffc72caf5c561c27a0e5f41816e14 Original author: song Review repair source: 544fa8bec2602f5fa323a8027cd8db9d31b0b230 Validated tree: 0976453d5fd2592d741a6e83d29a2d366c0fd15e Original history: codex/research-split-archive-20261020 Co-authored-by: Codex Signed-off-by: song Signed-off-by: Codex --- benchmark/runtime/checkpoint_wake.py | 16 +- benchmark/runtime/worker.py | 62 ++- benchmark/tests/test_resume_sessions.py | 111 ++++++ benchmark/tests/test_task_entry.py | 2 + .../integrations/runtime-connector-catalog.md | 103 +++++ loopx/chat_agent.py | 140 +++++-- loopx/cli_commands/turn.py | 5 + loopx/cli_commands/turn_decision.py | 128 ++++++ loopx/cli_commands/turn_run_once.py | 25 +- .../goals/first_party_host_admission.py | 158 +++++++- .../goals/first_party_host_runtime.ts | 61 ++- .../goals/global_registry_health.py | 51 ++- loopx/control_plane/turn_driver/codex_cli.py | 7 +- .../turn_driver/codex_operation_host.py | 6 + .../turn_driver/host_process_transport.py | 10 + loopx/extensions/process_runtime.py | 47 +++ .../test_benchmark_checkpoint_admission.py | 261 ++++++++++++ .../test_benchmark_current_admission.py | 168 ++++++++ .../test_codex_operation_launch.py | 317 +++++++++++++++ tests/control_plane/test_contract_health.py | 53 +++ .../test_first_party_host_runtime.py | 136 +++++++ .../test_host_current_admission.py | 96 +++++ .../control_plane/test_host_plan_read_only.py | 100 +++++ tests/control_plane/test_host_process.py | 51 +++ .../test_host_research_recovery_cli.py | 260 ++++++++++++ .../test_initial_planning_host_cli.py | 136 +++++++ .../test_phase_probe_continuation.py | 312 +++++++++++++++ .../content_digest_single_owner.test.ts | 1 + .../first_party_host_runtime.test.ts | 75 +++- tests/test_codex_operation_host.py | 47 ++- .../test_codex_operation_launch_admission.py | 88 ++++ tests/test_host_launch_authority.py | 375 ++++++++++++++++++ tests/test_loopx_turn_codex_cli.py | 215 +++++++++- 33 files changed, 3548 insertions(+), 75 deletions(-) create mode 100644 tests/control_plane/test_benchmark_checkpoint_admission.py create mode 100644 tests/control_plane/test_benchmark_current_admission.py create mode 100644 tests/control_plane/test_codex_operation_launch.py create mode 100644 tests/control_plane/test_host_current_admission.py create mode 100644 tests/control_plane/test_host_plan_read_only.py create mode 100644 tests/control_plane/test_host_research_recovery_cli.py create mode 100644 tests/control_plane/test_initial_planning_host_cli.py create mode 100644 tests/control_plane/test_phase_probe_continuation.py create mode 100644 tests/test_codex_operation_launch_admission.py create mode 100644 tests/test_host_launch_authority.py diff --git a/benchmark/runtime/checkpoint_wake.py b/benchmark/runtime/checkpoint_wake.py index 928bd62fb9..31ceb51398 100644 --- a/benchmark/runtime/checkpoint_wake.py +++ b/benchmark/runtime/checkpoint_wake.py @@ -184,17 +184,22 @@ def _capture(self, held, deadline): }, timeout=max(.1, window - time.time()), retry_safe=False, large_local_snapshot=True) def run(self, execution, body, wake, session_wake, stdout, stderr, timeout): - from .worker import child_process, native_command + from .worker import child_process, heartbeat_launch_check, native_command from .sessions import BenchmarkSessionWake deadline = time.monotonic() + timeout if timeout is not None else float("inf") + launch_env, original_quota = self.env, None while True: remaining = deadline - time.monotonic() if remaining <= 0: raise subprocess.TimeoutExpired("checkpoint source wake", timeout) - command = native_command(self.env, execution, "execute", wake, session_wake) + command = native_command(launch_env, execution, "execute", wake, session_wake) continuation = None - with child_process(command, env=self.env, stdout=stdout, stderr=stderr) as process: + with child_process(command, env=launch_env, stdout=stdout, stderr=stderr, + goal_admission=session_wake.admission if session_wake is not None else None, + check_current_launch=heartbeat_launch_check( + launch_env, session_wake, original_quota=original_quota)) as process: + self.receipt["host_invoked"] = True process.stdin.write(body) process.stdin.close() while process.poll() is None: @@ -252,4 +257,9 @@ def run(self, execution, body, wake, session_wake, stdout, stderr, timeout): session_wake = BenchmarkSessionWake(self.env, execution, self.receipt) if not source_session_id or session_wake.session_id != source_session_id: raise ValueError("checkpoint source continuation lost its native session") + # The held CLI guard can belong to another Turn inside this wake. + # Retain its typed packet, never recover identity from prompt text + # or reuse the enclosing wake's ID for its remaining settlement. + original_quota = held["quota"] + launch_env = {**self.env, "LOOPX_TURN": original_quota["heartbeat_receipt"]["turn_instance_id"]} body = continuation diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index 1ddeef2bc3..91dfbd553e 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -10,7 +10,7 @@ import subprocess import time import uuid -from contextlib import ExitStack, contextmanager +from contextlib import ExitStack, contextmanager, nullcontext from dataclasses import replace from pathlib import Path @@ -91,21 +91,48 @@ def heartbeat_body(env: dict[str, str], turn_id: str, *, native_goal: bool, @contextmanager -def child_process(command: list[str], *, env: dict[str, str], stdout, stderr): - process = subprocess.Popen( - command, - cwd=env["LOOPX_PROJECT"], - env=env, - stdin=subprocess.PIPE, - stdout=stdout, - stderr=stderr, - text=True, - start_new_session=True, - ) +def child_process(command: list[str], *, env: dict[str, str], stdout, stderr, + goal_admission=None, check_current_launch=None): + process = None try: + if check_current_launch is not None and goal_admission is None: + raise ValueError("current Host launch check requires its original Goal admission") + if check_current_launch is not None: + goal_admission = goal_admission.with_launch_decision(check_current_launch) + # Session selection may precede rendering/capture. Serialize only actual + # launch with the existing Goal lifetime owner, never the whole model run. + with (goal_admission.launch_lifetime(operation="codex_agent_host_launch") + if goal_admission is not None else nullcontext()): + process = subprocess.Popen( + command, + cwd=env["LOOPX_PROJECT"], + env=env, + stdin=subprocess.PIPE, + stdout=stdout, + stderr=stderr, + text=True, + start_new_session=True, + ) yield process finally: - _reap_child_process(process) + if process is not None: + _reap_child_process(process) + + +def heartbeat_launch_check(env, session_wake, *, original_quota=None): + """Carry the Host's real Turn inputs into the product's current quota owner.""" + if session_wake is None: + return None # Plain/native-Goal transports have no LoopX heartbeat owner. + from loopx.cli_commands.turn_decision import current_heartbeat_host_launch_decision + + def check(registry_path): + return current_heartbeat_host_launch_decision( + registry_path=registry_path, runtime_root=Path(env["LOOPX_RUNTIME_ROOT"]), + project=Path(env["LOOPX_PROJECT"]), goal_id=env["LOOPX_GOAL_ID"], + agent_id=env["LOOPX_AGENT_ID"], turn_instance_id=env["LOOPX_TURN"], + original_quota=original_quota, goal_ref=session_wake.goal_ref, + ) + return check def _reap_child_process(process): @@ -430,6 +457,7 @@ def run_once(env: dict[str, str]) -> dict: from benchmark.runtime.sessions import BenchmarkSessionWake session_wake = BenchmarkSessionWake(env, execution, receipt) + receipt["host_invoked"] = False if stage == "execute" and execution.mode == "heartbeat" and env.get("LOOPX_WARM_START"): from benchmark.runtime.checkpoint_session_refresh import prepare_session_refresh @@ -473,7 +501,11 @@ def run_once(env: dict[str, str]) -> dict: execution, body, wake, session_wake, stdout, stderr, timeout) receipt["ok"] = receipt["return_code"] == 0 else: - with child_process(command, env=env, stdout=stdout, stderr=stderr) as process: + with child_process(command, env=env, stdout=stdout, stderr=stderr, + goal_admission=session_wake.admission if session_wake is not None else None, + check_current_launch=heartbeat_launch_check(env, session_wake)) as process: + if session_wake is not None: + receipt["host_invoked"] = True process.communicate(input=body, timeout=timeout) receipt["return_code"] = process.returncode receipt["ok"] = process.returncode == 0 @@ -511,7 +543,7 @@ def run_once(env: dict[str, str]) -> dict: finally: checkpoint_context.close() try: - if session_wake is not None: + if session_wake is not None and receipt.get("host_invoked") is True: session_wake.observe(wake / "stdout.jsonl") except BaseException as exc: receipt.update(ok=False, **_error_tail(exc)) diff --git a/benchmark/tests/test_resume_sessions.py b/benchmark/tests/test_resume_sessions.py index adf58001d1..67562e6aa2 100644 --- a/benchmark/tests/test_resume_sessions.py +++ b/benchmark/tests/test_resume_sessions.py @@ -16,6 +16,48 @@ from loopx.control_plane.turn_driver.codex_sessions import load_codex_cli_session +@pytest.mark.skipif(os.name != "posix", reason="POSIX owned Host process groups") +def test_launch_admission_exit_interrupt_reaps_started_host(tmp_path, monkeypatch): + from contextlib import contextmanager + import subprocess + import sys + + from benchmark.runtime import worker + + class Admission: + @contextmanager + def launch_lifetime(self, *, operation): + yield + raise KeyboardInterrupt("interrupted while releasing launch admission") + + children = [] + popen = subprocess.Popen + + def record_child(*args, **kwargs): + process = popen(*args, **kwargs) + children.append(process) + return process + + monkeypatch.setattr(worker.subprocess, "Popen", record_child) + try: + with pytest.raises(KeyboardInterrupt, match="releasing launch admission"): + with worker.child_process( + [sys.executable, "-c", "import time; time.sleep(60)"], + env={**os.environ, "LOOPX_PROJECT": str(tmp_path)}, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + goal_admission=Admission(), + ): + pytest.fail("interrupted admission must not yield the Host") + assert len(children) == 1 + assert children[0].poll() is not None + with pytest.raises(ProcessLookupError): + os.killpg(children[0].pid, 0) + finally: + for process in children: + worker._reap_child_process(process) + process.stdin.close() + + @pytest.mark.parametrize("existing", ["valid", "malformed", "dangling"]) def test_absent_only_session_admission_preserves_every_existing_binding(tmp_path, existing): """The no-replacement condition is checked under the session owner's lock.""" @@ -88,10 +130,21 @@ def resume_env(tmp_path, monkeypatch): LOOPX_SHARED_SKILLS=str(skills), LOOPX_CLI="loopx", ) + Path(env["LOOPX_REGISTRY"]).write_text(json.dumps({ + "schema_version": "0.2", "projects": [], "goals": [{ + "id": env["LOOPX_GOAL_ID"], "status": "active", "repo": str(tmp_path), + "coordination": {"registered_agents": [env["LOOPX_AGENT_ID"]]}, + }], + })) monkeypatch.setattr( "benchmark.runtime.worker.heartbeat_body", lambda *a, **kw: "Current Todo input.", ) + # This fixture isolates native conversation/lifetime transport, including + # deliberately unregistered identities and lifecycle-only source profiles. + # Real current quota/health qualification is exercised by the CLI-backed + # test_benchmark_current_admission and checkpoint_admission integration tests. + monkeypatch.setattr("benchmark.runtime.worker.heartbeat_launch_check", lambda *a, **kw: None) Path(env["CODEX_BIN"]).write_text( f"#!{__import__('sys').executable}\n" + """ @@ -208,6 +261,57 @@ def communicate(*, input, timeout): assert second["session"]["session_id"] == first["session"]["session_id"] +@pytest.mark.parametrize("change", ["stop", "replace"]) +@pytest.mark.parametrize("profile", ["legacy", "source"]) +def test_heartbeat_rechecks_selected_session_at_native_launch(tmp_path, monkeypatch, change, profile): + from benchmark.runtime import worker + from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected + from loopx.control_plane.projects.registry_codec import project_registry_transaction, source_session_registry_transaction + from tests.test_loopx_turn_codex_cli import _source_admission + + env = resume_env(tmp_path, monkeypatch) + if profile == "source": + admission = _source_admission(tmp_path) + env["LOOPX_REGISTRY"] = str(admission.registry_path) + registry_path = Path(env["LOOPX_REGISTRY"]) + registry_transaction = source_session_registry_transaction if profile == "source" else project_registry_transaction + first = run_once(env) + original_binding = binding(env) + history = next((tmp_path / "logs/sessions").glob("*.jsonl")) + original_history = history.read_bytes() + native_command = worker.native_command + + def changed_after_selection(*args, **kwargs): + command = native_command(*args, **kwargs) + with registry_transaction( + registry_path, operation="fixture_stop_before_native_launch", + ) as transaction: + registry = transaction.payload_copy() + if change == "replace": + if profile == "source": + registry["goals"][0]["goal_instance_id"] = "ginst_" + "b" * 32 + else: + registry["goals"][0]["repo"] = str(tmp_path / "replacement") + else: + registry["goals"][0]["activation"] = { + "schema_version": "loopx_goal_activation_v1", "state": "stopped", + "updated_at": "2026-10-10T12:00:00Z", "reason": "Owner stopped this Goal", + } + transaction.commit(registry) + return command + + monkeypatch.setattr(worker, "native_command", changed_after_selection) + with pytest.raises(FirstPartyHostRuntimeRejected) as error: + run_once(env) + assert error.value.code == ("goal_stopped" if change == "stop" else + "stale_goal_instance" if profile == "source" else "stale_goal_authority") + receipt = json.loads(sorted((tmp_path / "logs/wakes").glob("*/receipt.json"))[-1].read_bytes()) + assert receipt["host_invoked"] is False and not receipt["ok"] + assert receipt["error_kind"] == "FirstPartyHostRuntimeRejected" + assert receipt["session"]["session_id"] == first["session"]["session_id"] + assert binding(env) == original_binding and history.read_bytes() == original_history + + @pytest.mark.parametrize("mutation", ["corrupt", "home", "fork"]) def test_resume_refuses_silent_replacement(tmp_path, monkeypatch, mutation): env = resume_env(tmp_path, monkeypatch) @@ -236,6 +340,13 @@ def test_another_identity_starts_an_independent_conversation( ): env = resume_env(tmp_path, monkeypatch) first = run_once(env) + if field == "LOOPX_GOAL_ID": + from loopx.control_plane.projects.registry_codec import project_registry_transaction + + with project_registry_transaction(Path(env["LOOPX_REGISTRY"]), operation="fixture_second_goal") as transaction: + payload = transaction.payload_copy() + payload["goals"].append({**payload["goals"][0], "id": "another-identity"}) + transaction.commit(payload) second = run_once(env | {field: "another-identity"}) assert second["session"]["action"] == "start_new" assert second["session"]["session_id"] != first["session"]["session_id"] diff --git a/benchmark/tests/test_task_entry.py b/benchmark/tests/test_task_entry.py index 51ee3f5bb9..bf21ff3332 100644 --- a/benchmark/tests/test_task_entry.py +++ b/benchmark/tests/test_task_entry.py @@ -70,8 +70,10 @@ def planning_env(tmp_path): { "id": "planning-goal", "repo": str(project), + "domain": "project", "state_file": str(state), "status": "active", + "adapter": {"kind": "read_only_project_map_v0", "status": "connected-read-only"}, "coordination": {"registered_agents": ["planner"]}, } ], diff --git a/docs/integrations/runtime-connector-catalog.md b/docs/integrations/runtime-connector-catalog.md index fb974b6e54..529a071e91 100644 --- a/docs/integrations/runtime-connector-catalog.md +++ b/docs/integrations/runtime-connector-catalog.md @@ -91,6 +91,109 @@ receipts or infer work completion from stdout. Stop signals and subprocess cleanup keep their existing behavior. This adapter does not override admission or reinterpret task eligibility. +First-party Codex CLI and benchmark Hosts also recheck their captured Goal +authority at the actual process-launch boundary, including checkpoint resumes. +The operation-equipped Codex app-server path uses the same admission at process +creation and again before sending `turn/start`. The existing authority writer +lock covers each launch/send boundary and is released before waiting for a +provider response. A Goal stopped after session selection therefore cannot start +a new Host or replace its saved session with an empty, unlaunched result. + +For operation-host `turn/start`, native stdin delivery has a one-second ceiling +and also respects any shorter original response/idle/hard deadline. Waiting +for another native writer consumes that allowance before acquiring the Goal +fence. Backpressure or an interrupted partial send cancels and reaps the same +native peer and drains the writer before releasing the fence; it cannot leave +a sender to deliver that Turn after Stop. This is a dispatch timeout, not proof +of non-acceptance or permission to replay. Successful large and Unicode inputs +retain the full original text stream; no new input-size truncation is applied. + +Source-session admission retains its exact Goal lifetime checks. Executable +legacy registries instead compare their original registry byte witness under +the existing registry writer lock and reuse the registry owner's legal active +states, including `active-read-only`. This does not relax source-session status +or identity checks. A changed registry requires a fresh plan, including when an unrelated registry +configuration change invalidated the old witness. When a legacy global +projection declares its source registry, launch checks both the original source +and the projection in the existing source-then-projection writer-lock order. +A committed source stop still rejects launch if projection synchronization +failed and left an active mirror. Missing or cyclic source routes cannot grant +launch permission. This conservative check does +not create an exact Goal instance identity: byte-identical legacy replacement +remains indistinguishable. Generic shell commands still require their own Host +to provide an equivalent launch boundary. + +Codex operation tools 路径在创建 app-server 和发送 `turn/start` 前复用同一 +准入检查;只在这两个边界持有原 owner 的锁,等待模型响应前释放。这样 Stop +既不会被另一条 Host 路径绕过,也不会被长时间持锁拖延。legacy 的合法 +`active-*` 状态复用原注册表规则,source-session 的生命周期校验保持严格。 +`turn/start` 的原生 stdin 发送最多占用一秒,并遵守更短的原 response、idle、hard +期限;等待其他原生写入也计入该界限。背压或部分发送中断时,先取消并回收同一个 +原生进程、等待写入线程退出,再释放 Goal 锁,避免 Stop 后迟到派发。这类超时不 +证明请求从未受理,也不授权重放;正常的大输入和 Unicode 文本完整保留,不截断。 + +The executable `turn run-once` path also reruns its existing current quota and +health decision under those same locks, against the actual source registry and +the original Turn/Todo/replan selection. It creates no replacement receipt. +Both plain exec and operation-tools launch obey that owner's refusal when +mirror quota is stale or source state becomes unavailable. Benchmark heartbeat +execution and heartbeat/turn initial planning reuse this quota owner before +native process creation. Initial planning does not need a prior heartbeat +receipt or Todo. An interrupted checkpoint source continuation carries its +actual held guard's Turn and nonempty event ID into the same current decision, +preserving the complete original settlement identity when present. It does not +derive these inputs from prompt text or reuse the enclosing wake's Turn. +The native session continues with the original selection and settlement rules. + +The existing health owner checks the current registry's declared `state_file` +even when no global registry is available. A missing declared state therefore +reaches the quota decision on local source routes; undeclared initial setup +does not create a missing-state finding, and a global copy of the same finding +is deduplicated. Launch follows that owner's decision, including any lawful +health-repair action, rather than treating every health finding as an +unconditional refusal. + +The launch reread retains the required inbox reads already dispatched for that +same Turn. They cover only the material observed by that dispatch; later material +still passes through the current quota owner's decision. If launch is refused, +completed preflight synchronization remains reported, while the refusal itself +creates no model Turn or quota debit. + +This full decision callback is qualified for executable legacy local, source +and source-to-global routes. Source-session admission retains its separate +lifecycle-only boundary; this qualification does not enable its CLI execution. +Benchmark `plain`, `native-goal` and `loopx-goal` modes keep their existing owners +and are outside this quota callback. Restoring an archived checkpoint uses the +destination Goal's current guard, without importing the source Goal's claims, +Turn or spend authority. A Todo-bound guard that owes closeout returns to its +original process without capture or restart; completing that work is not a new +checkpoint opportunity. + +可执行的 `turn run-once` 在同一锁范围内,使用真实 source 注册表重新运行既有 +quota/health 决策,并保持原 Turn、Todo、重规划选择;不会创建替代回执。 +原 quota/health owner 拒绝当前工作时,普通 exec 与 operation tools 都不能借 +旧镜像额度或新出现的源状态缺失启动。 +benchmark 的 heartbeat 执行,以及 heartbeat/turn 初始规划,也在创建原生进程 +前复用同一 quota owner;初始规划不要求已有 heartbeat 回执或 Todo。同一次 wake +中被中断的 checkpoint 源会话,直接携带被截留 guard 的原 Turn、非空 event ID +和已有的完整结算身份重新读取当前资格,不从提示文字提取身份,也不改用外层 wake 的 +Turn;原生会话按原选择和结算规则继续。 +当前注册表已声明的 `state_file` 由原健康检查 owner 核验,即使没有 global 注册表, +状态缺失也会进入 quota 决策。尚未声明状态的初始设置不会因此产生缺失报告,global +中的同一报告按原规则去重。启动遵守原 owner 的决定,也保留其允许的健康修复动作, +不把所有健康异常都改成无条件拒绝。 + +启动前重读保留同一 Turn 已派发的 inbox 必读结果,覆盖范围仅限该次派发实际 +观察到的材料;之后新增的材料仍由当前 quota owner 判断。启动被拒绝时,已完成 +的前置同步仍须如实报告;拒绝本身不会创建模型 Turn 或配额扣减。 + +完整决策验证覆盖当前可执行的 legacy 本地、source 和 source/global 路径。 +source-session 保留独立的生命周期边界,这项验证不会启用其 CLI 执行;benchmark +的 `plain`、`native-goal`、`loopx-goal` 模式继续使用各自原 owner,不属于这个 quota +回调。从归档 checkpoint 恢复时,使用目的 Goal 的当前 guard,不导入源 Goal 的 +claim、Turn 或扣额权限。已有待结算义务的 Todo-bound guard 直接返回原进程继续 +收尾,不捕获或重启;完成这项工作不构成新的 checkpoint 机会。 + ## External Tool Extension Candidates An MCP server can play two different roles around LoopX: diff --git a/loopx/chat_agent.py b/loopx/chat_agent.py index 03f7ac60cd..fae30630a3 100644 --- a/loopx/chat_agent.py +++ b/loopx/chat_agent.py @@ -10,7 +10,7 @@ import tempfile import threading import time -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable, Iterator @@ -686,6 +686,9 @@ class CodexChatAgentSession: _command_items_lock: threading.Lock = field(default_factory=threading.Lock, repr=False) _host_model_auth: Any = field(default=None, repr=False) _public_source_read_context: str = field(default="", repr=False) + _launch_guard: Callable[[], AbstractContextManager[None]] | None = field( + default=None, repr=False + ) @classmethod def start( @@ -709,6 +712,7 @@ def start( reasoning_effort: str | None = None, dynamic_tools: list[dict[str, Any]] | None = None, host_config: dict[str, Any] | None = None, + launch_guard: Callable[[], AbstractContextManager[None]] | None = None, _compatibility_catalog_path: Path | None = None, ) -> "CodexChatAgentSession": resolved = shutil.which(codex_bin) @@ -816,27 +820,38 @@ def start( ] ) command.extend(["--listen", "stdio://"]) + process = None try: - process = subprocess.Popen( - command, - cwd=str(root), - env=runtime_env, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - encoding="utf-8", - bufsize=1, - start_new_session=isolate_process_tree and os.name == "posix", - ) - except OSError as exc: - raise CodexChatAgentError( - "Codex app-server could not start", - gate=_host_tool_gate( - "Codex app-server could not start for LoopX Chat.", - "Verify the Codex CLI installation and app-server support.", - ), - ) from exc + # The admitting caller owns this short launch fence. Native startup + # RPCs and model execution must not hold the Goal writer lock. + with launch_guard() if launch_guard is not None else nullcontext(): + try: + process = subprocess.Popen( + command, + cwd=str(root), + env=runtime_env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + encoding="utf-8", + bufsize=1, + start_new_session=isolate_process_tree and os.name == "posix", + ) + except OSError as exc: + raise CodexChatAgentError( + "Codex app-server could not start", + gate=_host_tool_gate( + "Codex app-server could not start for LoopX Chat.", + "Verify the Codex CLI installation and app-server support.", + ), + ) from exc + except BaseException: + # A guard can fail on exit after Popen succeeded, before a Session + # exists to own cleanup. Reap that same process on this path too. + if process is not None: + cls._close_process(process, process_tree_owned=isolate_process_tree) + raise messages: "queue.Queue[dict[str, Any] | Exception]" = queue.Queue() assert process.stdout is not None @@ -862,6 +877,7 @@ def start( model=model, reasoning_effort=reasoning_effort, model_catalog_compatibility_applied=_compatibility_catalog_path is not None, + _launch_guard=launch_guard, ) try: session._request( @@ -1067,6 +1083,7 @@ def start( reasoning_effort=reasoning_effort, dynamic_tools=dynamic_tools, host_config=host_config, + launch_guard=launch_guard, _compatibility_catalog_path=catalog_path, ) except Exception: @@ -1082,18 +1099,55 @@ def _runtime_error(self, summary: str) -> CodexChatAgentError: ), ) - def _write(self, payload: dict[str, Any]) -> None: + def _write( + self, payload: dict[str, Any], *, + write_guard: Callable[[], AbstractContextManager[None]] | None = None, + deadline: float | None = None, + ) -> None: if self.process.stdin is None: raise self._runtime_error("Codex app-server input stream is closed.") + frame = json.dumps(payload, ensure_ascii=False) + "\n" try: + if write_guard is not None: + self._guarded_write(frame, write_guard, deadline=deadline) + return with self._write_lock: - self.process.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") + self.process.stdin.write(frame) self.process.stdin.flush() except (BrokenPipeError, OSError) as exc: raise self._runtime_error( "Codex app-server input stream closed unexpectedly." ) from exc + def _guarded_write( + self, frame: str, write_guard: Callable[[], AbstractContextManager[None]], *, + deadline: float | None, + ) -> None: + from .extensions.process_runtime import write_process_stdin_before_deadline + + # One frame has at most one second under the Goal's launch fence. + # A shorter original RPC/Turn deadline still wins. Waiting for another + # native writer spends this allowance before any authority is acquired. + dispatch_deadline = min( + time.monotonic() + 1.0, + deadline if deadline is not None else time.monotonic() + self.response_timeout_sec, + ) + if not self._write_lock.acquire(timeout=max(0, dispatch_deadline - time.monotonic())): + raise self._timeout_error("response_timeout", "Codex input dispatch timed out.") + try: + with write_guard(): + try: + write_process_stdin_before_deadline( + self.process, frame, deadline=dispatch_deadline, + cancel_and_reap=self.close, + ) + except TimeoutError as exc: + raise self._timeout_error( + "response_timeout", "Codex input dispatch timed out." + ) from exc + finally: + self._write_lock.release() + def _notify(self, method: str, params: dict[str, Any]) -> None: self._write({"method": method, "params": params}) @@ -1248,6 +1302,7 @@ def _request( *, request_id: int | None = None, deadline: float | None = None, + write_guard: Callable[[], AbstractContextManager[None]] | None = None, ) -> dict[str, Any]: if request_id is None: with self._request_id_lock: @@ -1257,9 +1312,16 @@ def _request( with self._response_waiters_lock: self._response_waiters[request_id] = waiter try: - self._write({"id": request_id, "method": method, "params": params}) response_deadline = time.monotonic() + self.response_timeout_sec deadline = min(deadline, response_deadline) if deadline is not None else response_deadline + # A per-Turn guard replaces the session default. Acquire it only + # inside the actual write, after waiting for the native write lock. + if write_guard is None and method == "turn/start": + write_guard = self._launch_guard + self._write( + {"id": request_id, "method": method, "params": params}, + **({"write_guard": write_guard, "deadline": deadline} if write_guard is not None else {}), + ) while True: if time.monotonic() >= deadline: raise self._timeout_error("response_timeout", "Codex app-server timed out.") @@ -1621,6 +1683,7 @@ def send( on_event: Callable[[str, dict[str, Any]], None] | None = None, output_schema: dict[str, Any] | None = None, on_native_item: Callable[[dict[str, Any]], None] | None = None, + turn_start_guard: Callable[[], AbstractContextManager[None]] | None = None, ) -> dict[str, Any]: text = " ".join(str(user_message or "").split()) if not text: @@ -1649,6 +1712,12 @@ def send( image_url = str(attachment.get("data_url") or "") if image_url: turn_input.append({"type": "image", "url": image_url, "detail": "auto"}) + # A running app-server is not permission for a new native Turn. Hold + # the caller's fence through its write, then release before waiting for + # the native acknowledgement or model output. + started_at = time.monotonic() + start_timeout = min(self.idle_timeout_sec, self.hard_timeout_sec + if self.hard_timeout_sec is not None else self.idle_timeout_sec) turn_result = self._request( "turn/start", { @@ -1664,6 +1733,8 @@ def send( ), }, request_id=request_id, + deadline=started_at + start_timeout, + **({"write_guard": turn_start_guard} if turn_start_guard is not None else {}), ) turn_id = _extract_id(turn_result, "turn", "turnId") self.current_turn_id = turn_id @@ -1676,8 +1747,7 @@ def send( display_filter = VisibleResponseStreamFilter(protected_paths=[self.work_dir]) steps = CodexActivitySteps(protected_paths=[self.work_dir]) visible_delta_count = 0 - started_at = time.monotonic() - last_activity_at = started_at + last_activity_at = time.monotonic() while True: now = time.monotonic() if self.hard_timeout_sec is not None and now - started_at >= self.hard_timeout_sec: @@ -1885,19 +1955,23 @@ def _timeout_error(self, error_code: str, summary: str) -> CodexChatTimeoutError ) def close(self) -> None: - if self.process_tree_owned: + self._close_process(self.process, process_tree_owned=self.process_tree_owned) + + @staticmethod + def _close_process(process: subprocess.Popen[str], *, process_tree_owned: bool) -> None: + if process_tree_owned: from .extensions.process_runtime import terminate_process_tree - terminate_process_tree(self.process, grace_seconds=0.1) + terminate_process_tree(process, grace_seconds=0.1) return - if self.process.poll() is not None: + if process.poll() is not None: return - self.process.terminate() + process.terminate() try: - self.process.wait(timeout=2) + process.wait(timeout=2) except subprocess.TimeoutExpired: - self.process.kill() - self.process.wait(timeout=2) + process.kill() + process.wait(timeout=2) def __enter__(self) -> "CodexChatAgentSession": return self diff --git a/loopx/cli_commands/turn.py b/loopx/cli_commands/turn.py index a58721a49e..c6c05e1684 100644 --- a/loopx/cli_commands/turn.py +++ b/loopx/cli_commands/turn.py @@ -85,6 +85,10 @@ def handle_turn_command( registry_path=registry_path, goal_id=args.goal_id, planned_goal_ref=goal_ref, + # This caller retains only source-profile admission. Legacy Host + # runners capture their launch witness when execution is requested; + # inspecting a plan must not create their registry writer lock. + capture_launch_authority=False, ) strict_goal_admission = goal_admission if goal_admission.enabled else None if getattr(args, "codex_operation_tools", False) and args.host != "codex-cli": @@ -146,6 +150,7 @@ def handle_turn_command( registry_path=registry_path, goal_id=args.goal_id, planned_goal_ref=payload.get("goal_ref"), + capture_launch_authority=False, ) strict_goal_admission = goal_admission if goal_admission.enabled else None if strict_goal_admission is not None: diff --git a/loopx/cli_commands/turn_decision.py b/loopx/cli_commands/turn_decision.py index 1c6da03601..9afe869ff2 100644 --- a/loopx/cli_commands/turn_decision.py +++ b/loopx/cli_commands/turn_decision.py @@ -31,6 +31,9 @@ ) from ..control_plane.quota.live_decision import build_live_quota_should_run_decision from ..control_plane.quota.turn_envelope import build_turn_envelope +from ..control_plane.quota.settlement import SettlementIdentity +from ..control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected +from ..control_plane.turn_driver.driver import build_loopx_turn_plan, selected_turn_todo from ..control_plane.scheduler.execution_context import ( scheduler_execution_context_for_turn, ) @@ -250,6 +253,131 @@ def build_fresh_turn_decision_owner( ) +def current_turn_host_launch_decision( + args: argparse.Namespace, *, registry_path: Path, runtime_root: Path, + identity: SettlementIdentity, replan_obligation_id: str | None, + planned_effective_action: object, + scheduler_execution_context: Mapping[str, Any], + operator_inbox_urgency_projector: Callable[..., dict[str, Any]], + turn_start_hook_dispatch: Mapping[str, Any] | None = None, + goal_ref: Mapping[str, object] | None = None, +) -> dict[str, Any]: + """Re-read the original Turn's eligibility inside its launch lifetime. + + The caller supplies the real receipt identity; a Host session or turn_key + cannot reconstruct it. This read neither creates another receipt nor + substitutes a newly selected task into the already planned Host request. + """ + current = build_live_quota_should_run_decision( + collect_turn_status_payload(args, registry_path=registry_path, + runtime_root_arg=str(runtime_root)), + goal_id=identity.goal_id, agent_id=identity.agent_id, + available_capabilities=args.available_capabilities, include_scheduler_detail=False, + codex_app_current_rrule=None, registry_path=registry_path, runtime_root=runtime_root, + route_source=TURN_DECISION_ROUTE_SOURCE, + scheduler_execution_context=scheduler_execution_context, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + bounded_research_frontier_projector=project_live_explore_composition_frontier, + receipt_bound_todo_id=identity.todo_id or None, + receipt_bound_replan_obligation_id=replan_obligation_id, + turn_instance_id=identity.turn_instance_id, + turn_start_hook_dispatch=turn_start_hook_dispatch, + workspace_path=Path(args.project).expanduser().resolve(), goal_ref=goal_ref, + interaction_projection_hooks=(periodic_report_pending_intent_interaction_hook( + registry_path=registry_path, runtime_root=runtime_root, + goal_id=identity.goal_id, agent_id=identity.agent_id, + ),), + ) + launch = {"current": current, "planned_effective_action": planned_effective_action} + # The shared typed launch owner retains refusal precedence (including + # Stop/witness failures) and compares the original effective action. + if (current.get("should_run") is not True + or current.get("effective_action") != planned_effective_action): + return launch + current_envelope = build_turn_envelope( + current, scheduler_execution_context=scheduler_execution_context, + ) + # Reuse the existing side-effect-free Host route projection, including + # delivery, user-gate and protocol checks. No plan/session is persisted. + route = build_loopx_turn_plan( + current_envelope, host=args.host, execution_mode=args.execution_mode, + scheduler_owner=args.scheduler_owner, turn_instance_id=identity.turn_instance_id, + goal_ref=goal_ref, + )["route"] + if route.get("would_invoke_host") is not True: + raise FirstPartyHostRuntimeRejected("goal_launch_not_allowed") + if selected_turn_todo(current_envelope).get("todo_id") != identity.todo_id: + raise FirstPartyHostRuntimeRejected("current_turn_selection_changed") + return launch + + +def current_heartbeat_host_launch_decision( + *, registry_path: Path, runtime_root: Path, project: Path, + goal_id: str, agent_id: str, turn_instance_id: str, + original_quota: Mapping[str, Any] | None = None, + goal_ref: Mapping[str, object] | None = None, +) -> dict[str, Any]: + """Re-read one native heartbeat/initial-plan launch without minting a receipt. + + Unlike an executing governed Turn, a first heartbeat or planning wake can + legitimately have no chosen Todo and no committed guard yet. Reuse the + public quota owner's original-selection reader and full live decision. A + checkpoint continuation additionally retains its exact held packet; its + committed binding cannot be replaced with the enclosing wake's identity. + Call only inside the Goal launch lifetime, immediately before native spawn. + """ + from ..control_plane.quota.heartbeat_receipt import heartbeat_receipt_view + from ..control_plane.scheduler.execution_context import scheduler_execution_context_for_runtime_profile + from ..turn_identity import normalize_turn_instance_id + from .quota_action_selection import load_requested_quota_action_selection + + original_turn = normalize_turn_instance_id(turn_instance_id) + if original_turn is None: + raise FirstPartyHostRuntimeRejected("heartbeat_launch_turn_missing") + args = argparse.Namespace( + goal_id=goal_id, agent_id=agent_id, turn_instance_id=original_turn, + todo_id=None, codex_app=False, runtime_profile="generic_cli", + scan_path=[str(project)], scan_root=str(project), limit=0, + available_capabilities=["shell", "filesystem_write"], + ) + selection = load_requested_quota_action_selection( + args, runtime_root=runtime_root, turn_instance_id=original_turn, goal_ref=goal_ref, + ) + if original_quota is not None: + held = original_quota.get("heartbeat_receipt") + if (original_quota.get("goal_id") != goal_id or not isinstance(held, Mapping) + or held.get("turn_instance_id") != original_turn or selection.receipt is None): + raise FirstPartyHostRuntimeRejected("heartbeat_launch_original_receipt_missing") + current_receipt = heartbeat_receipt_view( + selection.receipt, turn_instance_id=original_turn, status="replayed", + ) + if (not held.get("event_id") or current_receipt.get("event_id") != held.get("event_id") + or current_receipt.get("settlement_identity") != held.get("settlement_identity")): + raise FirstPartyHostRuntimeRejected("heartbeat_launch_original_binding_changed") + current = build_live_quota_should_run_decision( + collect_turn_status_payload(args, registry_path=registry_path, runtime_root_arg=str(runtime_root)), + goal_id=goal_id, agent_id=agent_id, available_capabilities=args.available_capabilities, + include_scheduler_detail=False, codex_app_current_rrule=None, + registry_path=registry_path, runtime_root=runtime_root, + scheduler_execution_context=scheduler_execution_context_for_runtime_profile("generic_cli"), + operator_inbox_urgency_projector=build_lark_operator_inbox_urgency_projector(runtime_root_arg=runtime_root), + bounded_research_frontier_projector=project_live_explore_composition_frontier, + receipt_bound_todo_id=selection.receipt_bound_todo_id, + receipt_bound_replan_obligation_id=selection.receipt_bound_replan_obligation_id, + retained_action_selection_todo_id=selection.retained_todo_id_for_decision, + turn_instance_id=original_turn, workspace_path=project, goal_ref=goal_ref, + interaction_projection_hooks=(periodic_report_pending_intent_interaction_hook( + registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, agent_id=agent_id, + ),), + ) + # Initial heartbeat/planning has not chosen an action yet. Its generic + # prompt asks the Host to consume the current decision after launch. A + # held continuation already has an action and cannot substitute another. + return {"current": current, "planned_effective_action": ( + original_quota.get("effective_action") if original_quota is not None + else current.get("effective_action"))} + + def fresh_turn_envelope( decision: Mapping[str, Any], *, diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index e5c8810f90..1bd89a5eb6 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -50,7 +50,7 @@ from ..state_refresh import refresh_state_run from ..control_plane.todos.path_resolution import resolve_todo_state_path from .turn_cadence import managed_cadence_start -from .turn_decision import collect_turn_status_payload +from .turn_decision import collect_turn_status_payload, current_turn_host_launch_decision from .turn_dsh_host import build_dsh_host_runner, build_dsh_operation_host_runner from .turn_rendering import build_turn_error_payload from .turn_post_writeback import drain_committed_turn_post_writeback_hooks @@ -768,6 +768,21 @@ def scheduler(_spend_payload: dict[str, object]) -> dict[str, object]: session_binding_resolver = None if args.host == "codex-cli": + def current_launch_decision(current_registry: Path) -> Mapping[str, Any]: + # Revalidate the already admitted identity through the same + # owner. In a global route the source is checked first, even + # when its mirror still advertises old quota or health. + return current_turn_host_launch_decision( + args, registry_path=current_registry, runtime_root=runtime_root, + identity=settlement_identity, replan_obligation_id=replan_obligation_id, + planned_effective_action=payload["turn_envelope"].get("effective_action"), + scheduler_execution_context=payload.get("scheduler_execution_context") or {}, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + # Keep this Turn's already dispatched required reads at launch. + turn_start_hook_dispatch=payload.get("turn_start_capability_hook_dispatch"), + goal_ref=goal_ref, + ) + def run_built_in_host( request: Mapping[str, Any], ) -> dict[str, Any]: @@ -781,8 +796,12 @@ def run_built_in_host( "mcp_server": args.codex_mcp_server_json, "timeout_seconds": None if args.timeout_seconds is None else max(1.0, args.timeout_seconds - 5.0), } - if goal_admission is not None: - options["goal_admission"] = goal_admission + options["goal_admission"] = ( + goal_admission or FirstPartyHostGoalAdmission.for_plan( + registry_path=registry_path, goal_id=args.goal_id, + planned_goal_ref=goal_ref, + ) + ).with_launch_decision(current_launch_decision) if getattr(args, "codex_operation_tools", False): from ..control_plane.turn_driver.codex_operation_host import ( run_codex_operation_host, diff --git a/loopx/control_plane/goals/first_party_host_admission.py b/loopx/control_plane/goals/first_party_host_admission.py index 1fb3941ee0..e0d6c17aa6 100644 --- a/loopx/control_plane/goals/first_party_host_admission.py +++ b/loopx/control_plane/goals/first_party_host_admission.py @@ -1,8 +1,8 @@ from __future__ import annotations from collections.abc import Callable, Iterator, Mapping -from contextlib import contextmanager -from dataclasses import dataclass +from contextlib import AbstractContextManager, ExitStack, contextmanager, nullcontext +from dataclasses import dataclass, replace from pathlib import Path from typing import Any, TypeVar @@ -15,6 +15,7 @@ from ..projects.registry_codec import ( SOURCE_SESSION_PROFILE_ID, load_project_registry, + project_registry_transaction, ) from ..effect_program import TurnProviderStepKind from .activation import goal_activation_state @@ -228,6 +229,43 @@ def capture_first_party_host_goal_ref( ) +def _legacy_launch_authority(registry: Mapping[str, Any], goal_id: str) -> dict[str, Any]: + goals = registry.get("goals") + if (registry.get("profile_id") == SOURCE_SESSION_PROFILE_ID or not isinstance(goals, list) + or any(not isinstance(goal, Mapping) for goal in goals)): + return {"kind": "unavailable"} + matches = [goal for goal in goals if goal.get("id") == goal_id] + if len(matches) != 1: + return {"kind": "absent"} + goal = matches[0] + try: + activation = goal_activation_state(goal).value + except ValueError: + return {"kind": "unavailable"} + return {"kind": "present", "goal_id": goal_id, "activation_state": activation, + "registration_status": goal.get("status", "active")} + + +def _legacy_source_registry( + registry: Mapping[str, Any], goal_id: str, requested_registry: Path, +) -> Path | None: + goals = registry.get("goals") + if not isinstance(goals, list): + return None + matches = [goal for goal in goals if isinstance(goal, Mapping) and goal.get("id") == goal_id] + if len(matches) != 1: + return None + source = matches[0].get("source_registry") + if source is None or source == "": + return None + if not isinstance(source, str) or not source.strip(): + raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") + resolved = Path(source.strip()).expanduser().resolve() + if resolved == requested_registry: + raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") + return resolved + + @dataclass(frozen=True, slots=True) class FirstPartyHostGoalAdmission: """Keep exact Host-state I/O inside one source Goal lifetime guard.""" @@ -236,6 +274,9 @@ class FirstPartyHostGoalAdmission: goal_id: str planned_goal_ref: object | None source_profile: bool + legacy_registry_witness: Mapping[str, str] | None = None + legacy_source_admission: FirstPartyHostGoalAdmission | None = None + launch_decision_reader: Callable[[Path], Mapping[str, Any]] | None = None @classmethod def for_plan( @@ -244,31 +285,105 @@ def for_plan( registry_path: Path, goal_id: str, planned_goal_ref: object | None, + capture_launch_authority: bool = True, ) -> FirstPartyHostGoalAdmission: requested_registry = registry_path.expanduser().resolve() require_goal_id(goal_id) source_profile = planned_goal_ref is not None + legacy_registry_witness = None + legacy_source_admission = None if not source_profile and requested_registry.is_file(): source_profile = ( load_project_registry(requested_registry).get("profile_id") == SOURCE_SESSION_PROFILE_ID ) + if not source_profile and capture_launch_authority: + with project_registry_transaction( + requested_registry, operation="first_party_host_launch_capture", + ) as transaction: + legacy_registry_witness = transaction.source_read_witness() + source_registry = _legacy_source_registry( + transaction.payload_copy(), goal_id, requested_registry, + ) + if source_registry is not None: + try: + with project_registry_transaction( + source_registry, operation="first_party_host_launch_source_capture", + ) as transaction: + # The existing source route names its owning project, + # not another mirror. Reject cycles/chains before any + # nested lifetime lock, including a self reference. + if _legacy_source_registry(transaction.payload_copy(), goal_id, source_registry) is not None: + raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") + legacy_source_admission = cls( + registry_path=source_registry, goal_id=goal_id, + planned_goal_ref=None, source_profile=False, + legacy_registry_witness=transaction.source_read_witness(), + ) + except (OSError, TypeError, ValueError) as exc: + raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") from exc return cls( registry_path=requested_registry, goal_id=goal_id, planned_goal_ref=planned_goal_ref, source_profile=source_profile, + legacy_registry_witness=legacy_registry_witness, + legacy_source_admission=legacy_source_admission, ) + @classmethod + def launch_scope_for_plan( + cls, *, admission: FirstPartyHostGoalAdmission | None, + registry_path: Path | None, goal_id: str, planned_goal_ref: object | None, + ) -> Callable[[str], AbstractContextManager[None]]: + """Capture once, then reuse the same admission at each native boundary. + + A provided Turn admission keeps its quota/health reader. Legacy callers + capture their original registry witness here; callers without a registry + retain their existing unmanaged transport without inventing authority. + """ + if admission is None and registry_path is not None: + admission = cls.for_plan(registry_path=registry_path, goal_id=goal_id, + planned_goal_ref=planned_goal_ref) + + def scope(operation: str) -> AbstractContextManager[None]: + return admission.launch_lifetime(operation=operation) if admission is not None else nullcontext() + + return scope + @property def enabled(self) -> bool: return self.source_profile + def with_launch_decision( + self, reader: Callable[[Path], Mapping[str, Any]], + ) -> FirstPartyHostGoalAdmission: + """Bind a legacy Turn's existing read-only quota owner to launch. + + Source and mirror must each read their own current authority. The + reader retains the original Turn/Todo/settlement identity; it cannot + replace the plan or issue a receipt while checking it. + """ + if self.source_profile: + # The lifecycle-only profile has its existing exact lifetime seam; + # runtime admission rejects it before business/Host execution. + # Its quota reader takes index -> source locks and cannot be + # nested inside that lifetime's source guard. + return self + return replace( + self, launch_decision_reader=reader, + legacy_source_admission=(self.legacy_source_admission.with_launch_decision(reader) + if self.legacy_source_admission is not None else None), + ) + def _decision( self, operation: str, *, host_state: Mapping[str, Any] | None = None, + launch_authority: Mapping[str, Any] | None = None, + registry_witness: Mapping[str, str] | None = None, + launch_decision: Mapping[str, Any] | None = None, ) -> dict[str, Any]: decision = effect_runtime_result( "goal.first_party_host_runtime.decide", @@ -278,11 +393,15 @@ def _decision( ), "operation": operation, "planned_goal_ref": self.planned_goal_ref, - "authority": source_goal_authority( + "authority": dict(launch_authority) if launch_authority is not None else source_goal_authority( self.registry_path, self.goal_id, ), **({"host_state": host_state} if host_state is not None else {}), + **({"launch_decision": dict(launch_decision)} if launch_decision is not None else {}), + **({"goal_id": self.goal_id, "registry_path": str(self.registry_path), + "planned_registry_witness": self.legacy_registry_witness, + "registry_witness": dict(registry_witness)} if registry_witness is not None else {}), }, ) if not isinstance(decision, dict): @@ -344,6 +463,39 @@ def current_lifetime(self, *, operation: str) -> Iterator[None]: self._decision("require_current") yield + @contextmanager + def launch_lifetime(self, *, operation: str) -> Iterator[None]: + """Recheck the original launch authority under its existing writer lock. + + Legacy registries have no exact Goal instance identity. Their original + byte witness can reject stale admission without promoting an alias to + one; a changed registry requires a fresh plan. A caller with the original + Turn binding supplies its full current admission check. This lifecycle + witness alone does not certify quota/health. Hold only through dispatch. + """ + if self.source_profile: + with self.current_lifetime(operation=operation): + yield + return + with ExitStack() as scope: + if self.legacy_source_admission is not None: + # Activation writes source then global projection under these + # same locks. A failed mirror sync must not hide a source stop. + scope.enter_context(self.legacy_source_admission.launch_lifetime(operation=operation)) + try: + transaction = scope.enter_context(project_registry_transaction( + self.registry_path, operation=operation, + )) + except (OSError, TypeError, ValueError) as exc: + raise FirstPartyHostRuntimeRejected("goal_authority_unavailable") from exc + self._decision( + "launch", launch_authority=_legacy_launch_authority(transaction.payload_copy(), self.goal_id), + registry_witness=transaction.source_read_witness(), + launch_decision=(self.launch_decision_reader(self.registry_path) + if self.launch_decision_reader is not None else None), + ) + yield + @contextmanager def source_journal_admission( self, diff --git a/loopx/control_plane/goals/first_party_host_runtime.ts b/loopx/control_plane/goals/first_party_host_runtime.ts index 5e0867db43..ad5d8eb943 100644 --- a/loopx/control_plane/goals/first_party_host_runtime.ts +++ b/loopx/control_plane/goals/first_party_host_runtime.ts @@ -1,6 +1,7 @@ import type { JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; import { assertNever, jsonObject } from "../runtime_decode.ts"; +import { BARE_SHA256_PATTERN } from "../content_digest.ts"; import { parseExactGoalRef, type ExactGoalRef, @@ -13,7 +14,7 @@ type WireGoalRef = Readonly<{ goal_instance_id: string; }>; -type Operation = "select_state" | "require_current" | "accept_result"; +type Operation = "select_state" | "require_current" | "accept_result" | "launch"; type RejectionCode = | "goal_not_registered" @@ -21,6 +22,8 @@ type RejectionCode = | "goal_instance_id_missing" | "goal_stopped" | "stale_goal_instance" + | "stale_goal_authority" + | "goal_launch_not_allowed" | "legacy_host_state" | "host_state_goal_instance_mismatch" | "host_state_malformed"; @@ -47,6 +50,7 @@ function requiredOperation(value: unknown): Operation { value === "select_state" || value === "require_current" || value === "accept_result" + || value === "launch" ) { return value; } @@ -129,10 +133,60 @@ function selectHostState( return { kind: "resume", goal_ref: wireGoalRef(plannedGoalRef) }; } +function registryWitness(value: unknown): {path: string; sha256: string} | null { + const witness = jsonObject(value); + return witness && typeof witness.path === "string" && !!witness.path + && typeof witness.sha256 === "string" && BARE_SHA256_PATTERN.test(witness.sha256) + ? {path: witness.path, sha256: witness.sha256} : null; +} + +/** A legacy launch rechecks its captured registry; it gains no GoalRef. */ +function legacyLaunch(facts: JsonObject): FirstPartyHostRuntimeDecision { + const current = jsonObject(facts.authority); + if (current?.kind === "absent") return {kind: "reject", code: "goal_not_registered"}; + if (current?.kind !== "present" || typeof facts.goal_id !== "string" || !facts.goal_id + || current.goal_id !== facts.goal_id) return {kind: "reject", code: "goal_authority_unavailable"}; + // Legacy registration accepts the active-* variants used by the existing + // Goal frontier. Source-session registration remains exact-only below. + const registration = typeof current.registration_status === "string" + ? current.registration_status.trim().toLowerCase() : ""; + if (registration !== "active" && !registration.startsWith("active-")) { + return {kind: "reject", code: "goal_not_registered"}; + } + if (current.activation_state === "stopped") return {kind: "reject", code: "goal_stopped"}; + if (current.activation_state !== "active") return {kind: "reject", code: "goal_authority_unavailable"}; + const planned = registryWitness(facts.planned_registry_witness); + const observed = registryWitness(facts.registry_witness); + if (!planned || !observed || planned.path !== facts.registry_path) { + return {kind: "reject", code: "goal_authority_unavailable"}; + } + if (planned.path !== observed.path || planned.sha256 !== observed.sha256) { + return {kind: "reject", code: "stale_goal_authority"}; + } + const rejected = launchDecisionRejection(facts); + if (rejected) return rejected; + return {kind: "legacy"}; +} + +function launchDecisionRejection(facts: JsonObject): FirstPartyHostRuntimeDecision | null { + if (!("launch_decision" in facts)) return null; + const launch = jsonObject(facts.launch_decision); + const decision = jsonObject(launch?.current); + // Consume the current quota owner's decision; do not reproduce its quota, + // health, repair, replan or scheduler policy in the Host adapter. + // A newly selected repair/replan is not permission to launch the old action. + // An already-admitted health repair keeps the quota owner's exception. + return decision?.should_run === true && typeof decision.status_health_ok === "boolean" + && typeof launch?.planned_effective_action === "string" && !!launch.planned_effective_action + && decision.effective_action === launch.planned_effective_action + ? null : {kind: "reject", code: "goal_launch_not_allowed"}; +} + /** * Decide whether one first-party Host may select state or accept a result. * - * Non-source profiles retain their historical behavior. The source-session + * Non-source state I/O retains its historical behavior; Host launch separately + * requires a current captured registry witness. The source-session * profile is exact-only: neither an alias-only plan nor alias-only Host state * can be upgraded by inference. */ @@ -146,7 +200,7 @@ export function decideFirstPartyHostRuntime( ); } if (facts.profile_id !== SOURCE_SESSION_PROFILE_ID) { - return { kind: "legacy" }; + return facts.operation === "launch" ? legacyLaunch(facts) : { kind: "legacy" }; } const operation = requiredOperation(facts.operation); @@ -178,6 +232,7 @@ export function decideFirstPartyHostRuntime( case "select_state": return selectHostState(facts.host_state, planned.value); case "require_current": + case "launch": return { kind: "resume", goal_ref: wireGoalRef(planned.value) }; case "accept_result": return { kind: "accept_result", goal_ref: wireGoalRef(planned.value) }; diff --git a/loopx/control_plane/goals/global_registry_health.py b/loopx/control_plane/goals/global_registry_health.py index b72adee40f..eaa2990121 100644 --- a/loopx/control_plane/goals/global_registry_health.py +++ b/loopx/control_plane/goals/global_registry_health.py @@ -44,6 +44,28 @@ def collect_global_registry_health( parse_timestamp: Callable[[Any], datetime | None], ) -> dict[str, Any]: global_path = global_registry_path(runtime_root) + current_goals = registry_goals(current_registry) + current_state_findings: list[dict[str, Any]] = [] + declared_state_count = 0 + for goal in current_goals: + state_path = resolve_goal_local_path( + goal.get("state_file"), goal, fallback_base=registry_path.parent, + ) + if state_path is None: + # Initial setup without a declared state has no missing-state fact. + continue + declared_state_count += 1 + if not state_path.exists(): + goal_id = str(goal.get("id") or "unknown-goal") + current_state_findings.append(global_registry_finding( + kind="state_file_missing", severity="action", goal_id=goal_id, + path=state_path, message=f"`{goal_id}` active state file is missing", + recommended_action=f"repair `{goal_id}` state_file or reconnect the project", + )) + current_checks = ( + [f"current registry active state files checked: {declared_state_count}"] + if declared_state_count else [] + ) if not global_path.exists(): return { "available": False, @@ -52,9 +74,10 @@ def collect_global_registry_health( "registry": str(global_path), "current_registry": str(registry_path), "current_registry_is_global": False, - "summary": {"high": 0, "action": 0, "info": 0, "checks": 0, "findings": 0}, - "findings": [], - "checks": [], + "summary": {"high": 0, "action": len(current_state_findings), "info": 0, + "checks": len(current_checks), "findings": len(current_state_findings)}, + "findings": current_state_findings, + "checks": current_checks, } try: @@ -67,22 +90,22 @@ def collect_global_registry_health( "registry": str(global_path), "current_registry": str(registry_path), "current_registry_is_global": False, - "summary": {"high": 1, "action": 0, "info": 0, "checks": 1, "findings": 1}, + "summary": {"high": 1, "action": len(current_state_findings), "info": 0, + "checks": len(current_checks) + 1, "findings": len(current_state_findings) + 1}, "findings": [global_registry_finding( kind="global_registry_unreadable", severity="high", message="global membership could not be read", recommended_action="repair the global registry source and retry the projection", - )], - "checks": ["global registry readability"], + ), *current_state_findings], + "checks": ["global registry readability", *current_checks], } global_goals = registry_goals(global_registry) - current_goals = registry_goals(current_registry) current_ids = {str(goal.get("id")) for goal in current_goals if goal.get("id")} global_ids = [str(goal.get("id")) for goal in global_goals if goal.get("id")] global_id_set = set(global_ids) source_registries: set[str] = set() findings: list[dict[str, Any]] = [] - checks: list[str] = [] + checks: list[str] = list(current_checks) current_is_global = same_path(registry_path, global_path) id_counts = Counter(global_ids) @@ -177,6 +200,18 @@ def collect_global_registry_health( ) ) + # Keep global diagnostics (including orphan guidance) when the same state + # is represented in both views. A current-only or changed path is still a + # local health fact, independent of global membership or sync freshness. + for finding in current_state_findings: + if not any( + existing.get("kind") == "state_file_missing" + and existing.get("goal_id") == finding["goal_id"] + and same_path(Path(existing["path"]), Path(finding["path"])) + for existing in findings + ): + findings.append(finding) + missing_from_current = sorted(global_id_set - current_ids) if not current_is_global and missing_from_current: shown = missing_from_current[:8] diff --git a/loopx/control_plane/turn_driver/codex_cli.py b/loopx/control_plane/turn_driver/codex_cli.py index 6553f223b8..a129901bcc 100644 --- a/loopx/control_plane/turn_driver/codex_cli.py +++ b/loopx/control_plane/turn_driver/codex_cli.py @@ -670,6 +670,10 @@ def run_codex_cli_host( if not resolved or not Path(resolved).exists(): raise ValueError("Codex CLI executable is unavailable") lineage = _lineage(request) + launch_scope = FirstPartyHostGoalAdmission.launch_scope_for_plan( + admission=goal_admission, registry_path=registry_path, + goal_id=lineage["goal_id"], planned_goal_ref=request.get("goal_ref"), + ) planned_session = _mapping(request.get("session")) planned_action = str(planned_session.get("action") or "") context_policy = _mapping(planned_session.get("context_policy")) @@ -816,7 +820,8 @@ def observe_stderr(line: str) -> None: diagnostics = HostOutputLines(observe_stderr) observed = run_host_process(command, project=project, input_text=_prompt(request), timeout_seconds=timeout_seconds, drain_timeout_seconds=OUTPUT_DRAIN_TIMEOUT_SECONDS, - on_stdout=events.feed, on_stderr=diagnostics.feed) + on_stdout=events.feed, on_stderr=diagnostics.feed, + launch_guard=lambda: launch_scope("codex_cli_host_launch")) events.finish() diagnostics.finish() returncode = observed["returncode"] diff --git a/loopx/control_plane/turn_driver/codex_operation_host.py b/loopx/control_plane/turn_driver/codex_operation_host.py index 26cb34f08f..788de38159 100644 --- a/loopx/control_plane/turn_driver/codex_operation_host.py +++ b/loopx/control_plane/turn_driver/codex_operation_host.py @@ -296,6 +296,10 @@ def run_codex_operation_host( } if request_source is not None: profile["operation_request"] = request_source.profile(source_route) + launch_scope = FirstPartyHostGoalAdmission.launch_scope_for_plan( + admission=goal_admission, registry_path=registry_path, + goal_id=lineage["goal_id"], planned_goal_ref=request.get("goal_ref"), + ) profile_digest = hashlib.sha256( json.dumps(profile, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() @@ -380,6 +384,7 @@ def run_codex_operation_host( resume_thread_id=binding["session_id"] if binding else None, dynamic_tools=[OPERATION_TOOL], host_config=host_config, + launch_guard=lambda: launch_scope("codex_operation_process_launch"), response_timeout_sec=30 if timeout_seconds is None else min(timeout_seconds, 30), hard_timeout_sec=timeout_seconds, idle_timeout_sec=180 if timeout_seconds is None else min(timeout_seconds, 180), @@ -487,6 +492,7 @@ def record_child() -> None: if continuations else ""), output_schema=codex_cli_result_schema(request), on_event=on_event, + turn_start_guard=lambda: launch_scope("codex_operation_turn_start"), **({"on_native_item": observe_child} if child_observer is not None else {}), ) return result diff --git a/loopx/control_plane/turn_driver/host_process_transport.py b/loopx/control_plane/turn_driver/host_process_transport.py index 1431cc7f09..4a169e0a12 100644 --- a/loopx/control_plane/turn_driver/host_process_transport.py +++ b/loopx/control_plane/turn_driver/host_process_transport.py @@ -8,6 +8,7 @@ import sys import tempfile from collections.abc import Callable, Sequence +from contextlib import AbstractContextManager, ExitStack from pathlib import Path from typing import Any @@ -223,11 +224,14 @@ def run_host_process( on_stderr: Callable[[str], None] | None = None, delegated_lease: dict[str, Any] | None = None, environment: dict[str, str] | None = None, + launch_guard: Callable[[], AbstractContextManager[None]] | None = None, ) -> dict[str, Any]: """Keep the control pipe open until exit; EOF cancels the owned process group. Callbacks observe transient chunks. They must not persist raw Host output. TS owns deadlines, byte budgets, termination and the final observation. + An owning admission guard covers request delivery through spawn observation, + then releases before Host output; it must never lock the entire Host run. """ command = list(argv) if sys.platform == "win32": @@ -281,7 +285,10 @@ def run_host_process( assert proc.stdin is not None and proc.stdout is not None result = None record = None + launch_scope = ExitStack() try: + if launch_guard is not None: + launch_scope.enter_context(launch_guard()) if record_path is not None: # Written before the request, so no Host exists that the record # does not name; a record that cannot be written launches nothing. @@ -313,6 +320,7 @@ def run_host_process( record.update(phase="spawned", host_pid=event["pid"], process_group=group if isinstance(group, int) else None) _write_host_process_record(record_path, record) + launch_scope.close() elif kind in {"stdout", "stderr"} and isinstance(event.get("text"), str): consume = on_stdout if kind == "stdout" else on_stderr if consume is not None: @@ -325,9 +333,11 @@ def run_host_process( "spawn_failed", }: result = event + launch_scope.close() else: raise RuntimeError("Invalid managed Host process observation") finally: + launch_scope.close() # Also runs on callback failure / Ctrl-C. Do not kill the supervisor # before it has had a chance to terminate its owned Host group. proc.stdin.close() diff --git a/loopx/extensions/process_runtime.py b/loopx/extensions/process_runtime.py index 4a5dcfd026..4f9121420a 100644 --- a/loopx/extensions/process_runtime.py +++ b/loopx/extensions/process_runtime.py @@ -19,6 +19,53 @@ _PROCESS_GROUP_STOP_TIMEOUT_SECONDS = 1.0 +def write_process_stdin_before_deadline( + process: subprocess.Popen[str], text: str, *, deadline: float, + cancel_and_reap: Callable[[], None], +) -> None: + """Finish one native frame, or cancel its peer before returning failure. + + The caller owns the serialization and launch fence. A bounded wait keeps a + full pipe from stranding that fence; the same process is reaped and its + writer joined before the caller releases it. Use the existing text stream + so successful large/Unicode frames retain their original wire semantics on + every supported Python platform, including Windows 3.11. + """ + stream = process.stdin + if stream is None: + raise BrokenPipeError("native input stream is closed") + if time.monotonic() >= deadline: + raise TimeoutError("native input dispatch deadline expired") + finished = threading.Event() + failures: list[BaseException] = [] + + def write() -> None: + try: + stream.write(text) + stream.flush() + except BaseException as exc: + failures.append(exc) + finally: + finished.set() + + writer = threading.Thread(target=write, name="loopx-native-stdin-writer", daemon=True) + writer.start() + try: + if not finished.wait(max(0, deadline - time.monotonic())): + raise TimeoutError("native input dispatch deadline expired") + if failures: + raise failures[0] + except BaseException: + try: + cancel_and_reap() + finally: + writer.join(timeout=_PROCESS_TERMINATE_GRACE_SECONDS) + if writer.is_alive(): + raise RuntimeError("native input writer did not drain after peer cancellation") + raise + writer.join() + + @dataclass(frozen=True) class CappedProcessResult: returncode: int diff --git a/tests/control_plane/test_benchmark_checkpoint_admission.py b/tests/control_plane/test_benchmark_checkpoint_admission.py new file mode 100644 index 0000000000..e7c9d1a1d4 --- /dev/null +++ b/tests/control_plane/test_benchmark_checkpoint_admission.py @@ -0,0 +1,261 @@ +"""Actual checkpoint handoff retains the original Turn at every native resume.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import sys + +import pytest + +from benchmark.runtime import worker +from benchmark.runtime.checkpoint_wake import CheckpointWake +from benchmark.runtime.codex import Execution, prepare_codex_home, process_environment +from benchmark.runtime.sessions import BenchmarkSessionWake +from loopx.configure_goal import configure_goal +from loopx.control_plane.effect_runtime import effect_runtime_result +from loopx.control_plane.goals import activation_service +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected +from tests.control_plane import test_quota_settlement_cli as fixture +from tests.control_plane.host_process_fixture import write_python_host_fixture +from tests.control_plane.test_benchmark_current_admission import _environment, _quota +from tests.control_plane.test_codex_operation_launch import _bindings + + +# CheckpointWake's production transport interrupts an owned POSIX process group. +pytestmark = pytest.mark.skipif(os.name != "posix", reason="CheckpointWake requires Unix handoff and process groups") +HELD_TURN = "checkpoint-held-original-turn" +OUTER_TURN = "checkpoint-enclosing-wake" + +_HOST = r''' +import hashlib, json, os, shlex, subprocess, sys +from pathlib import Path +env = os.environ +count = Path("checkpoint-native.jsonl") +started = len(count.read_text().splitlines()) if count.exists() else 0 +with count.open("a") as stream: + stream.write(json.dumps({"argv":sys.argv[1:], "turn":env["LOOPX_TURN"]})+"\n") +body = sys.stdin.read() +with Path("checkpoint-model.jsonl").open("a") as stream: + stream.write(json.dumps({"turn":env["LOOPX_TURN"]})+"\n") +print(json.dumps({"type":"thread.started", "thread_id":"benchmark-original-session"}), flush=True) +base = [env["LOOPX_CLI"], "--format", "json", "--registry", env["LOOPX_REGISTRY"], + "--runtime-root", env["LOOPX_RUNTIME_ROOT"]] +def cli(*args): + result = subprocess.run([*base,*args], capture_output=True, text=True) + value = json.loads(result.stdout) + if result.returncode: + raise RuntimeError(json.dumps(value)) + return value +def rendered(command): + tokens = shlex.split(command) + index = next(i for i, token in enumerate(tokens) if token in {"quota","refresh-state"}) + turn = guard["heartbeat_receipt"]["turn_instance_id"] + return [turn if token == "${LOOPX_TURN:?}" else token for token in tokens[index:]] +if started == 0: + Path("retained-input.json").write_text(json.dumps([2,3])) + args = ["quota","should-run","--goal-id",env["LOOPX_GOAL_ID"],"--agent-id",env["LOOPX_AGENT_ID"], + "--runtime-profile","generic_cli","--turn-instance-id",env["FIXTURE_HELD_TURN"], + "--scan-path",env["LOOPX_PROJECT"],"--available-capability","shell", + "--available-capability","filesystem_write"] + if env.get("FIXTURE_BOUND_TODO"): + planning = cli(*args) + assert planning["heartbeat_receipt"]["settlement_binding_owed"] is True + args += ["--todo-id",env["FIXTURE_BOUND_TODO"]] + guard = cli(*args,"--decision-output-root",env["FIXTURE_CAPTURE_ROOT"], + "--decision-handoff-socket",env["FIXTURE_HANDOFF_SOCKET"]) + # A Todo-bound guard owes closeout. The existing selector releases its + # response to this same process; it is not a new capture opportunity. + assert env.get("FIXTURE_BOUND_TODO") and guard["heartbeat_receipt"]["closeout_required"] is True + Path("checkpoint-closeout-guard.json").write_text(json.dumps(guard)) +else: + assert started == 1 and "resume" in sys.argv and sys.argv[-2] == "benchmark-original-session" + guard = json.loads(Path(env["FIXTURE_HELD_QUOTA"]).read_text()) + assert env["LOOPX_TURN"] == guard["heartbeat_receipt"]["turn_instance_id"] + assert env["LOOPX_TURN"] != env["FIXTURE_OUTER_TURN"] + assert guard["heartbeat_receipt"]["event_id"] in body +assert guard["heartbeat_receipt"]["turn_instance_id"] == env["FIXTURE_HELD_TURN"] +Path("checkpoint-original-consumed.json").write_text(json.dumps(guard["heartbeat_receipt"])) +if guard["heartbeat_receipt"].get("settlement_identity") is None: + todo_id = env["FIXTURE_SELECTION_TODO"] + args = ["quota","should-run","--goal-id",env["LOOPX_GOAL_ID"],"--agent-id",env["LOOPX_AGENT_ID"], + "--runtime-profile","generic_cli","--turn-instance-id",env["LOOPX_TURN"],"--todo-id",todo_id, + "--scan-path",env["LOOPX_PROJECT"]] + selected = subprocess.run([*base,*args], capture_output=True, text=True) + guard = json.loads(selected.stdout) + if selected.returncode: + [recovery] = guard["interaction_contract"]["cli_channel"]["next_cli_actions"] + guard = cli(*rendered(recovery)) + assert guard["heartbeat_receipt"]["settlement_identity"]["todo_id"] == todo_id +Path("checkpoint-business.json").write_text(json.dumps({"sum":sum(json.loads(Path("retained-input.json").read_text()))})) +evidence = "checkpoint-business:" + hashlib.sha256(Path("checkpoint-business.json").read_bytes()).hexdigest() +vision = Path("checkpoint-vision.json").resolve() +vision.write_text(json.dumps({"schema_version":"goal_vision_replan_contract_v0","agent_id":env["LOOPX_AGENT_ID"], + "state":"active", "vision_patch":{"acceptance_summary":"Validate the retained inputs and keep remaining validation work.", + "advancement_policy":"as_needed"}, "path_delta":{"schema_version":"goal_path_delta_v0","outcome":"continue", + "prior_assumption":"The retained inputs still require a concrete validation probe.", + "observed_reality":"The retained inputs sum to five; this does not validate the other open tasks.", + "retained":["The remaining independent validation work"],"evidence_refs":[evidence]}})) +actions = guard["interaction_contract"]["cli_channel"]["next_cli_actions"] +refresh = next(command for command in actions if "refresh-state" in command) +for old,new in [("","advanced"), + ("","checkpoint-input"),("","retained-input-sum"), + ("","deterministic-sum"),("",evidence)]: + refresh = refresh.replace(old,new) +refresh_args = rendered(refresh) +if "--agent-vision-json" in refresh_args: + refresh_args[refresh_args.index("--agent-vision-json")+1] = str(vision) +else: + refresh_args += ["--agent-vision-json",str(vision)] +updated = cli(*refresh_args) +spend = rendered(next(command for command in actions if "spend-slot" in command)) +spent, replay = cli(*spend,"--scan-path",env["LOOPX_PROJECT"]), cli(*spend,"--scan-path",env["LOOPX_PROJECT"]) +Path("checkpoint-settlement.json").write_text(json.dumps({"refresh":updated,"spend":spent,"replay":replay})) +''' + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("refusal", ["source_compute_zero", "missing_state", "stop", "original_event_changed", None]) +def test_checkpoint_resume_rechecks_current_source_with_original_guard(tmp_path, monkeypatch, provider, refusal): + _run_checkpoint(tmp_path, monkeypatch, provider, refusal=refusal, todo_bound=False) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_checkpoint_todo_bound_closeout_keeps_original_host(tmp_path, monkeypatch, provider): + _run_checkpoint(tmp_path, monkeypatch, provider, refusal=None, todo_bound=True) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_checkpoint_unbound_guard_selects_existing_work_in_original_turn(tmp_path, monkeypatch, provider): + _run_checkpoint(tmp_path, monkeypatch, provider, refusal=None, todo_bound=False, unbound=True) + + +def _run_checkpoint(tmp_path, monkeypatch, provider, *, refusal, todo_bound, unbound=False): + unbound = unbound or refusal == "original_event_changed" + env, project, runtime, source, state = _environment( + tmp_path, monkeypatch, provider, replan="todo" if todo_bound or unbound else True) + env["CODEX_BIN"] = str(write_python_host_fixture(project / "checkpoint-host", f"#!{sys.executable}\n" + _HOST)) + env = process_environment(Path(env["LOOPX_CODEX_HOME"]), base=env) + env.update(LOOPX_TURN=OUTER_TURN, FIXTURE_OUTER_TURN=OUTER_TURN, FIXTURE_HELD_TURN=HELD_TURN) + if unbound: + env["FIXTURE_SELECTION_TODO"] = fixture.SELECTED_REPLAN_TODO_ID + if todo_bound: + # Select through the public guard inside the already running Host. + # Prebinding another Turn before this wake would correctly block the + # first launch as outstanding original work. Its Todo then owes + # closeout, so the selector must preserve this original process. + env["FIXTURE_BOUND_TODO"] = fixture.SELECTED_REPLAN_TODO_ID + execution = Execution(mode="heartbeat", context="resume", task_entry="seeded-todo", timeout_seconds=120) + prepare_codex_home(Path(env["LOOPX_CODEX_HOME"]), execution=execution, workspace=project, + model=env["MODEL_NAME"], effort=env["REASONING_EFFORT"], + base_url=env["OPENAI_BASE_URL"], api_key=env["OPENAI_API_KEY"], wire_api="responses", + skills=Path(env["LOOPX_SHARED_SKILLS"])) + log_root = Path(env["LOOPX_WAKE_LOG_DIR"]) + wake = log_root / OUTER_TURN + wake.mkdir(parents=True) + component = project / "checkpoint-source.txt" + component.write_text("Retained source before the intercepted original guard.\n") + request = project / "checkpoint-request.json" + request.write_text(json.dumps({"destination":str(log_root.parent / "checkpoints"), + "components":[{"id":"source","source_path":str(component)}], "point_metadata":{}, "trigger":"fresh_replan"})) + env["LOOPX_CHECKPOINT_REQUEST"] = str(request) + receipt = {"turn_id":OUTER_TURN,"mode":"heartbeat","stage":"execute"} + observations = {} + mirror_before = Path(env["LOOPX_REGISTRY"]).read_bytes() + with CheckpointWake(env, receipt) as hook: + env.update(FIXTURE_CAPTURE_ROOT=str(hook.capture_root), FIXTURE_HANDOFF_SOCKET=hook.socket_path) + body = worker.heartbeat_body(env, OUTER_TURN, native_goal=False, decision_handoff_socket=hook.socket_path) + session = BenchmarkSessionWake(env, execution, receipt) + original_capture = hook._capture + + def captured_then_changed(held, deadline): + captured = original_capture(held, deadline) + observations.update(held=held, bindings=_bindings(runtime), + events=fixture._heartbeat_receipt_events(runtime, HELD_TURN)) + if unbound: + assert held["quota"]["heartbeat_receipt"].get("settlement_identity") is None + assert held["quota"]["heartbeat_receipt"]["settlement_binding_owed"] is True + if todo_bound: + assert held["quota"]["heartbeat_receipt"]["settlement_identity"]["todo_id"] == fixture.SELECTED_REPLAN_TODO_ID + env["FIXTURE_HELD_QUOTA"] = str(held["path"]) + if refusal == "source_compute_zero": + changed = configure_goal(registry_path=source, goal_id=fixture.GOAL_ID, quota_compute=0, execute=True) + assert changed["written"], changed + assert _quota(source, runtime, project)["should_run"] is False + assert _quota(Path(env["LOOPX_REGISTRY"]), runtime, project)["should_run"] is True + elif refusal == "missing_state": + state.rename(state.with_suffix(".saved")) + assert _quota(source, runtime, project)["status_health_ok"] is False + elif refusal == "stop": + with monkeypatch.context() as fault: + fault.setattr(activation_service, "sync_project_registry_to_global", lambda **kwargs: { + "ok":False,"error":"fixture mirror sync failed after committed Stop"}) + changed = activation_service.set_goal_activation_state(registry_path=source, + goal_id=fixture.GOAL_ID, state="stopped", reason="Stop before checkpoint continuation.", + actor_kind="owner", execute=True) + assert changed["written"] and json.loads(source.read_text())["goals"][0]["activation"]["state"] == "stopped" + elif refusal == "original_event_changed": + original_receipt = held["quota"]["heartbeat_receipt"] + assert original_receipt.get("settlement_identity") is None and original_receipt["event_id"] + # A mismatched original packet cannot borrow the same Turn's + # unbound receipt merely because both have no selected Todo. + held["quota"] = {**held["quota"], "heartbeat_receipt": { + **original_receipt, "event_id": "different-committed-guard-event"}} + return captured + + monkeypatch.setattr(hook, "_capture", captured_then_changed) + error = None + with (wake / "stdout.jsonl").open("w") as stdout, (wake / "stderr.log").open("w") as stderr: + try: + returned = hook.run(execution, body, wake, session, stdout, stderr, 120) + except FirstPartyHostRuntimeRejected as rejected: + error = rejected + assert Path(env["LOOPX_REGISTRY"]).read_bytes() == mirror_before + assert not fixture._heartbeat_receipt_events(runtime, OUTER_TURN) + native = [json.loads(line) for line in (project / "checkpoint-native.jsonl").read_text().splitlines()] + models = [json.loads(line) for line in (project / "checkpoint-model.jsonl").read_text().splitlines()] + if todo_bound: + assert not observations and not receipt.get("checkpoint_handoffs"), receipt + assert not hook.root.exists(), "A closeout response created a fresh checkpoint attempt." + held = {"quota":json.loads((project / "checkpoint-closeout-guard.json").read_text())} + original = held["quota"]["heartbeat_receipt"] + assert original["closeout_required"] is True + assert original["settlement_identity"]["todo_id"] == fixture.SELECTED_REPLAN_TODO_ID + assert effect_runtime_result("runtime.checkpoint_artifacts", { + "action":"select_replan", "quota":held["quota"], "goal_id":fixture.GOAL_ID, + "agent_id":fixture.AGENT_ID, "turn_instance_id":HELD_TURN, + }, large_local_snapshot=True) == { + "capture":False, "reason":"turn_closeout_pending", "execution_authority_granted":False, + } + else: + assert observations and receipt["checkpoint_handoffs"][0]["captured"] is True, receipt + held = observations["held"] + assert _bindings(runtime) == observations["bindings"] + assert held["quota"]["heartbeat_receipt"]["turn_instance_id"] == HELD_TURN != OUTER_TURN + if refusal: + assert len(native) == len(models) == 1, "A refused original continuation launched another native Host." + assert error is not None and { + "source_compute_zero":"goal_launch_not_allowed", "missing_state":"goal_launch_not_allowed", + "stop":"goal_stopped", "original_event_changed":"heartbeat_launch_original_binding_changed", + }[refusal] in str(error) + assert not (project / "checkpoint-business.json").exists() + assert fixture._heartbeat_receipt_events(runtime, HELD_TURN) == observations["events"] + assert fixture._spend_run_count(runtime) == 0 + else: + assert error is None and returned == 0, (error, (wake / "stderr.log").read_text()) + expected_turns = [OUTER_TURN] if todo_bound else [OUTER_TURN, HELD_TURN] + assert len(models) == len(expected_turns) and [row["turn"] for row in native] == expected_turns + assert json.loads((project / "checkpoint-business.json").read_text()) == {"sum":5} + result = json.loads((project / "checkpoint-settlement.json").read_text()) + assert result["refresh"]["settlement_result"]["ok"] is True + assert result["spend"]["settlement_result"]["ok"] is True + assert result["replay"]["idempotent_replay"] is True and result["replay"]["appended"] is False + assert fixture._spend_run_count(runtime) == 1 + assert json.loads((project / "checkpoint-original-consumed.json").read_text()) == held["quota"]["heartbeat_receipt"] + events = fixture._heartbeat_receipt_events(runtime, HELD_TURN) + if todo_bound: + assert events[-1]["event_id"] == held["quota"]["heartbeat_receipt"]["event_id"] + else: + assert events[:len(observations["events"])] == observations["events"] + if not unbound: + assert events == observations["events"] diff --git a/tests/control_plane/test_benchmark_current_admission.py b/tests/control_plane/test_benchmark_current_admission.py new file mode 100644 index 0000000000..479238c8d6 --- /dev/null +++ b/tests/control_plane/test_benchmark_current_admission.py @@ -0,0 +1,168 @@ +"""Real heartbeat Hosts consume current source eligibility before native launch.""" +from __future__ import annotations + +import json +from pathlib import Path +import sys +import time + +import pytest + +from benchmark.runtime import worker +from benchmark.tests.test_shared_codex_runtime import worker_env +from loopx.configure_goal import configure_goal +from loopx.control_plane.goals import activation_service +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected +from loopx.global_registry import sync_project_registry_to_global +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests.control_plane.host_process_fixture import write_python_host_fixture +from tests.control_plane import test_quota_settlement_cli as fixture +from tests.control_plane.test_codex_operation_launch import _bindings + + +ROOT = Path(__file__).resolve().parents[2] + + +def _environment(tmp_path, monkeypatch, provider, *, replan=False): + isolate_sqlite_runtime(tmp_path, monkeypatch) + monkeypatch.setenv("LOOPX_USAGE_PING", "0") + monkeypatch.setenv("PYTHONPATH", str(ROOT)) + project, runtime, source = fixture._write_fixture(tmp_path) + if replan == "empty": + state = project / json.loads(source.read_text())["goals"][0]["state_file"] + state.write_text("---\nstatus: active\nowner_mode: goal\n" + 'objective: "Plan one concrete task from the current objective."\n---\n\n' + "# Active Goal State\n\n## Objective\n\nPlan one concrete task.\n\n" + "## Agent Todo\n\n## Completed Work Archive\n") + elif replan == "todo": + fixture._configure_selected_todo_replan_fixture(project, source) + elif replan: + fixture._configure_autonomous_replan_fixture(project, runtime, source) + fixture._initialize_fixture_authority(project, runtime, provider=provider) + synced = sync_project_registry_to_global( + registry_path=source, runtime_root_override=str(runtime), goal_id=fixture.GOAL_ID, + ) + assert synced["ok"], synced + env = worker_env(project) + skills = tmp_path / "shared-skills" + skills.mkdir() + env.update(LOOPX_EXECUTION_MODE="heartbeat", LOOPX_ITERATION_CONTEXT="resume", + LOOPX_TASK_ENTRY="seeded-todo", LOOPX_TASK_STAGE="execute", + LOOPX_RUNTIME_ROOT=str(runtime), LOOPX_REGISTRY=str(runtime / "registry.global.json"), + LOOPX_GOAL_ID=fixture.GOAL_ID, LOOPX_AGENT_ID=fixture.AGENT_ID, + LOOPX_SHARED_SKILLS=str(skills), + LOOPX_CODEX_TURN_TIMEOUT_SEC="90") + env["LOOPX_CLI"] = str(write_python_host_fixture(tmp_path / "loopx-cli", + f"#!{sys.executable}\nfrom loopx.cli import main\nraise SystemExit(main())\n")) + env["CODEX_BIN"] = str(write_python_host_fixture(Path(env["CODEX_BIN"]), f"#!{sys.executable}\n" + r''' +import json,sys +from pathlib import Path +with Path("native-starts").open("a") as marker: marker.write("native\n") +sys.stdin.read() +with Path("model-starts").open("a") as marker: marker.write("model\n") +Path("business.json").write_text(json.dumps({"value": sum([2,3])})) +print(json.dumps({"type":"thread.started","thread_id":"benchmark-original-session"}),flush=True) +''')) + state = project / json.loads(source.read_text())["goals"][0]["state_file"] + return env, project, runtime, source, state + + +def _quota(registry, runtime, project): + code, result = fixture._run_cli(registry, runtime, + "quota", "should-run", "--goal-id", fixture.GOAL_ID, "--agent-id", fixture.AGENT_ID, + "--runtime-profile", "generic_cli", cwd=project) + assert code == 0 or code == 1 and result.get("status_health_ok") is False, result + return result + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("refusal", ["source_compute_zero", "missing_state", "stop", None]) +def test_heartbeat_worker_qualifies_current_source_at_actual_spawn(tmp_path, monkeypatch, provider, refusal): + env, project, runtime, source, state = _environment(tmp_path, monkeypatch, provider) + mirror = Path(env["LOOPX_REGISTRY"]) + mirror_before, state_before = mirror.read_bytes(), state.read_bytes() + observations = [] + if refusal == "source_compute_zero": + changed = configure_goal(registry_path=source, goal_id=fixture.GOAL_ID, quota_compute=0, execute=True) + assert changed["written"], changed + denied = _quota(source, runtime, project) + assert denied["should_run"] is False and "compute_quota_zero" in json.dumps(denied), denied + assert _quota(mirror, runtime, project)["should_run"] is True + observations.append(denied) + original_command = worker.native_command + + def after_session_selection(*arguments): + if refusal == "missing_state": + state.rename(state.with_suffix(".saved")) + for registry in (source, mirror): + denied = _quota(registry, runtime, project) + assert denied.get("should_run") is not True and denied["status_health_ok"] is False, denied + observations.append(denied) + elif refusal == "stop": + with monkeypatch.context() as fault: + fault.setattr(activation_service, "sync_project_registry_to_global", lambda **kwargs: { + "ok": False, "error": "fixture mirror sync failed after source Stop committed"}) + stopped = activation_service.set_goal_activation_state( + registry_path=source, goal_id=fixture.GOAL_ID, state="stopped", + reason="Stop before the actual heartbeat model launch.", actor_kind="owner", execute=True) + assert stopped["written"] and json.loads(source.read_text())["goals"][0]["activation"]["state"] == "stopped" + observations.append(stopped) + return original_command(*arguments) + + monkeypatch.setattr(worker, "native_command", after_session_selection) + bindings_before = _bindings(runtime) + rejected = None + try: + result = worker.run_once(env) + except FirstPartyHostRuntimeRejected as error: + rejected = error + if refusal is None: + assert rejected is None and result["ok"], (rejected, result) + assert (project / "native-starts").read_text().splitlines() == ["native"] + assert (project / "model-starts").read_text().splitlines() == ["model"] + assert json.loads((project / "business.json").read_text()) == {"value": 5} + return + assert observations and mirror.read_bytes() == mirror_before + assert not (project / "native-starts").exists(), "Current refusal was bypassed by the real native process." + assert not (project / "model-starts").exists() and not (project / "business.json").exists() + assert rejected is not None + assert ("goal_stopped" if refusal == "stop" else "goal_launch_not_allowed") in str(rejected) + assert _bindings(runtime) == bindings_before + if refusal != "missing_state": + assert state.read_bytes() == state_before + index = runtime / "goals" / fixture.GOAL_ID / "runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("mode", ["heartbeat", "turn"]) +def test_initial_benchmark_planner_can_launch_without_a_prior_heartbeat_receipt(tmp_path, monkeypatch, provider, mode): + env, project, runtime, source, _state = _environment(tmp_path, monkeypatch, provider, replan="empty") + packet = _quota(source, runtime, project) + assert packet["should_run"] is True and packet.get("selected_todo") is None + env.update(LOOPX_TASK_ENTRY="loopx-planned", LOOPX_TASK_STAGE="plan", LOOPX_EXECUTION_MODE=mode, + LOOPX_PHASE_DEADLINE_EPOCH=str(time.time()+600), LOOPX_PLANNING_RESULT=str(project / "planning.json")) + if mode == "turn": + env["LOOPX_VALIDATION_COMMAND_JSON"] = json.dumps([sys.executable,"-c","pass"]) + source_code = r''' +import json, os, subprocess, sys +from pathlib import Path +packet = json.loads(sys.stdin.read().split("Host-supplied planning checkpoint:\n",1)[1]) +assert packet["runnable_todo_ids"] == [] +command = [os.environ["LOOPX_CLI"],"--format","json","--registry",os.environ["LOOPX_REGISTRY"], + "--runtime-root",os.environ["LOOPX_RUNTIME_ROOT"],"todo","add","--goal-id",os.environ["LOOPX_GOAL_ID"], + "--role","agent","--claimed-by",os.environ["LOOPX_AGENT_ID"],"--text","[P1] Validate the planned inputs.", + "--task-class","advancement_task","--action-kind","validate","--execute"] +added = json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout) +result = {"input_digest":packet["input_digest"],"status":"ready","todo_ids":[added["todo_id"]]} +Path(sys.argv[sys.argv.index("--output-last-message")+1]).write_text(json.dumps(result)) +Path("initial-plan.json").write_text(json.dumps(result)) +print(json.dumps({"type":"thread.started","thread_id":"initial-planning-session"}),flush=True) +''' + env["CODEX_BIN"] = str(write_python_host_fixture(project / "initial-planner", f"#!{sys.executable}\n"+source_code)) + result = worker.run_once(env) + assert result["ok"] and result["planning"]["state_readback_verified"] is True, result + planned = json.loads((project / "initial-plan.json").read_text()) + assert len(planned["todo_ids"]) == 1 and result["planning"]["todo_ids"] == planned["todo_ids"] + assert not fixture._heartbeat_receipt_events(runtime, result["turn_id"]) + assert fixture._spend_run_count(runtime) == 0 diff --git a/tests/control_plane/test_codex_operation_launch.py b/tests/control_plane/test_codex_operation_launch.py new file mode 100644 index 0000000000..d0388e61e6 --- /dev/null +++ b/tests/control_plane/test_codex_operation_launch.py @@ -0,0 +1,317 @@ +"""Actual operation-equipped CLI launch honors the original Goal admission.""" +from __future__ import annotations + +import contextlib +import io +import json +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from loopx.chat_agent import CodexChatAgentSession +from loopx.cli import main as cli_main +from loopx.control_plane.goals import activation_service +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission +from loopx.control_plane.projects.registry_codec import project_registry_transaction +from loopx.control_plane.turn_driver import codex_operation_host +from loopx.global_registry import sync_project_registry_to_global +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests.test_codex_operation_host import FAKE_SERVER +from tests.test_loopx_turn_codex_cli import _fake_codex, _request, _session_files +from tests.test_loopx_turn_driver import _write_live_fixture + + +def _fixture(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider) + native, turns = tmp_path / "native-starts", tmp_path / "native-turns" + executable = tmp_path / "fake-codex-operation" + executable.write_text(FAKE_SERVER.replace( + "import json, sys", "import json, sys, pathlib\n" + f"with pathlib.Path({str(native)!r}).open('a') as marker: marker.write('native\\n')", 1, + ).replace( + 'elif method == "turn/start":', 'elif method == "turn/start":\n' + f" with pathlib.Path({str(turns)!r}).open('a') as marker: marker.write(row['params']['threadId'] + '\\n')", 1, + )) + executable.chmod(0o700) + arguments = [ + "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", + "turn", "run-once", "--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture", + "--host", "codex-cli", "--project", str(project), "--scan-root", str(project), + "--no-global-sync", "--codex-operation-tools", "--codex-bin", str(executable), + "--codex-model", "test-model", "--codex-reasoning-effort", "xhigh", + "--codex-sandbox", "read-only", "--timeout-seconds", "30", + "--turn-instance-id", "operation-launch-fixture", + "--validation-command-json", json.dumps([sys.executable, "-c", "pass"]), + ] + return runtime, registry, native, turns, arguments + + +def _cli(arguments: list[str]) -> tuple[int, dict]: + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = cli_main(arguments) + return code, json.loads(output.getvalue()) + + +def _bindings(runtime: Path) -> dict[str, bytes]: + return {str(path.relative_to(runtime)): path.read_bytes() for path in _session_files(runtime)} + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("boundary", ["process", "turn"]) +@pytest.mark.parametrize("route", ["source", "mirror", "resume"]) +def test_operation_cli_stop_preempts_actual_native_boundary( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, boundary: str, route: str, +) -> None: + runtime, registry, native, turns, arguments = _fixture(tmp_path, monkeypatch, provider) + source = registry + original_mirror = None + if route == "mirror": + sync = sync_project_registry_to_global( + registry_path=source, runtime_root_override=str(runtime), goal_id="loopx-turn-fixture", + ) + assert sync["ok"], sync + registry = runtime / "registry.global.json" + original_mirror = registry.read_bytes() + arguments[arguments.index("--registry") + 1] = str(registry) + elif route == "resume": + # Establish the original native binding through the production owned + # transport; the later Stop still enters through actual CLI options. + request = _request() + request["turn_envelope"]["goal_id"] = "loopx-turn-fixture" + seeded = codex_operation_host.run_codex_operation_host( + request, runtime_root=runtime, registry_path=registry, project=source.parent.parent, + codex_bin=arguments[arguments.index("--codex-bin") + 1], + model="test-model", reasoning_effort="xhigh", timeout_seconds=30, + ) + assert seeded["result_kind"] == "wait" and _bindings(runtime) + before_native = len(native.read_text().splitlines()) if native.exists() else 0 + before_turns = turns.read_bytes() if turns.exists() else None + stopped = [] + original_bindings = [] + + def stop_goal() -> None: + original_bindings.append(_bindings(runtime)) + with monkeypatch.context() as fault: + if route == "mirror": + fault.setattr(activation_service, "sync_project_registry_to_global", lambda **kwargs: { + "ok": False, "error": "fixture mirror sync failed after source Stop committed", + }) + result = activation_service.set_goal_activation_state( + registry_path=registry, goal_id="loopx-turn-fixture", state="stopped", + reason="Owner stopped the admitted Goal before its native boundary.", + actor_kind="owner", execute=True, + ) + if route == "mirror": + assert result["written"] is True and result["partial_write"] is True, result + assert registry.read_bytes() == original_mirror + else: + assert result["ok"] and result["readback"]["verified"], result + assert json.loads(source.read_text())["goals"][0]["activation"]["state"] == "stopped" + stopped.append(result) + + if boundary == "process": + original_start = CodexChatAgentSession.start + + def stop_before_start(**kwargs): + stop_goal() + return original_start(**kwargs) + + monkeypatch.setattr(CodexChatAgentSession, "start", stop_before_start) + else: + original_prompt = codex_operation_host._prompt + + def stop_before_turn(request): + stop_goal() + return original_prompt(request) + + monkeypatch.setattr(codex_operation_host, "_prompt", stop_before_turn) + code, payload = _cli([*arguments, "--execute"]) + assert len(stopped) == 1, payload + assert (turns.read_bytes() if turns.exists() else None) == before_turns, payload + native_count = len(native.read_text().splitlines()) if native.exists() else 0 + assert native_count == before_native + (boundary == "turn"), payload + assert code == 1 and "goal_stopped" in json.dumps(payload), payload + assert _bindings(runtime) == original_bindings[0] + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_legacy_active_variant_cli_preview_and_real_launch_agree( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + runtime, registry, _, _, arguments = _fixture(tmp_path, monkeypatch, provider) + with project_registry_transaction(registry, operation="fixture_active_registration") as transaction: + payload = transaction.payload_copy() + payload["goals"][0]["status"] = "active-read-only" + transaction.commit(payload) + executable, native_log = _fake_codex(tmp_path) + monkeypatch.setenv("FAKE_CODEX_LOG", str(native_log)) + arguments.remove("--codex-operation-tools") + arguments[arguments.index("--codex-bin") + 1] = str(executable) + code, preview = _cli(arguments) + assert code == 0 and preview["effects"]["host_invoked"] is False, preview + code, executed = _cli([*arguments, "--execute"]) + assert code == 0 and executed["effects"]["host_invoked"] is True, executed + assert native_log.exists() and _bindings(runtime) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_operation_dispatch_releases_goal_lock_before_native_response_wait( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + runtime, registry, native, turns, arguments = _fixture(tmp_path, monkeypatch, provider) + release = tmp_path / "release-native-response" + executable = Path(arguments[arguments.index("--codex-bin") + 1]) + executable.write_text(executable.read_text().replace( + ' assert row["params"]["outputSchema"]["type"] == "object"', + f" import time\n while not pathlib.Path({str(release)!r}).exists(): time.sleep(.01)\n" + ' assert row["params"]["outputSchema"]["type"] == "object"', 1, + )) + with ThreadPoolExecutor(max_workers=2) as executor: + host = executor.submit(_cli, [*arguments, "--execute"]) + try: + deadline = time.monotonic() + 20 + while not turns.exists() and not host.done() and time.monotonic() < deadline: + time.sleep(.01) + assert turns.exists(), host.result() if host.done() else "native dispatch was not reached" + bindings = _bindings(runtime) + stop = executor.submit( + activation_service.set_goal_activation_state, + registry_path=registry, goal_id="loopx-turn-fixture", state="stopped", + reason="Owner Stop must complete while the original native response is pending.", + actor_kind="owner", execute=True, + ) + stopped = stop.result(timeout=5) + assert stopped["ok"] and stopped["readback"]["verified"], stopped + assert not host.done() and not release.exists() + finally: + release.write_text("continue the original fake native response") + host.result(timeout=30) + assert len(native.read_text().splitlines()) == 1 + assert turns.read_text().splitlines() == ["owned-app-server-thread"] + assert _bindings(runtime) == bindings + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +def test_native_process_is_reaped_if_launch_guard_exit_is_interrupted(tmp_path, monkeypatch): + import loopx.chat_agent as chat_agent + + executable = tmp_path / "fake-native-waiter" + executable.write_text(f"#!{sys.executable}\nimport sys\nsys.stdin.read()\n") + executable.chmod(0o700) + started = [] + popen = chat_agent.subprocess.Popen + + def observe_popen(*args, **kwargs): + process = popen(*args, **kwargs) + started.append(process) + return process + + @contextlib.contextmanager + def guard(): + yield + raise KeyboardInterrupt("fixture interrupted launch guard cleanup") + + monkeypatch.setattr(chat_agent.subprocess, "Popen", observe_popen) + with pytest.raises(KeyboardInterrupt, match="fixture interrupted"): + CodexChatAgentSession.start( + codex_bin=str(executable), work_dir=tmp_path, goal_id=None, objective="Synthetic cleanup", + isolate_process_tree=True, launch_guard=guard, + ) + assert len(started) == 1 and started[0].poll() is not None + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_backpressured_native_turn_drains_before_stop_writer_is_released( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + """A full stdin pipe cannot hold Stop or deliver a late native Turn.""" + runtime, registry, _, turns, arguments = _fixture(tmp_path, monkeypatch, provider) + release = tmp_path / "release-native-input" + executable = Path(arguments[arguments.index("--codex-bin") + 1]) + ready_response = ' emit({"id": row["id"], "result": {"thread": {"id": thread}, "model": "test-model", "reasoningEffort": "xhigh"}})' + executable.write_text(executable.read_text().replace( + ready_response, + ready_response + "\n import time\n" + f" while not pathlib.Path({str(release)!r}).exists(): time.sleep(.01)", + 1, + )) + arguments[arguments.index("--timeout-seconds") + 1] = "1" + original_prompt = codex_operation_host._prompt + monkeypatch.setattr(codex_operation_host, "_prompt", + lambda request: original_prompt(request) + "\n" + "x" * (8 * 1024 * 1024)) + sessions, drained_before_release = [], [] + entered = threading.Event() + original_start = CodexChatAgentSession.start + original_lifetime = FirstPartyHostGoalAdmission.launch_lifetime + + def start(**kwargs): + session = original_start(**kwargs) + sessions.append(session) + return session + + @contextlib.contextmanager + def lifetime(self, *, operation): + with original_lifetime(self, operation=operation): + if operation != "codex_operation_turn_start": + yield + return + entered.set() + try: + yield + finally: + drained_before_release.append(sessions[0].process.poll() is not None) + + monkeypatch.setattr(CodexChatAgentSession, "start", start) + monkeypatch.setattr(FirstPartyHostGoalAdmission, "launch_lifetime", lifetime) + with ThreadPoolExecutor(max_workers=2) as executor: + host = executor.submit(_cli, [*arguments, "--execute"]) + try: + assert entered.wait(timeout=30), host.result() if host.done() else "native launch not reached" + began = time.monotonic() + stop = executor.submit( + activation_service.set_goal_activation_state, + registry_path=registry, goal_id="loopx-turn-fixture", state="stopped", + reason="Synthetic Stop while native stdin is backpressured.", + actor_kind="owner", execute=True, + ) + stopped = stop.result(timeout=5) + code, payload = host.result(timeout=5) + elapsed = time.monotonic() - began + assert stopped["ok"] and stopped["readback"]["verified"], stopped + assert code == 1 and "timeout" in json.dumps(payload), payload + assert elapsed < 3, elapsed # One-second send plus finite peer cleanup. + assert drained_before_release == [True] + assert sessions[0].process.poll() is not None + assert not turns.exists() and not payload["effects"]["quota_spent"], payload + finally: + # This only unblocks a broken implementation's fixture teardown. + # The passing path proved cancellation before the peer may read. + release.touch() + if sessions: + sessions[0].close() + assert not turns.exists() + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +def test_guarded_native_dispatch_preserves_large_unicode_input(tmp_path, monkeypatch): + _, _, _, turns, arguments = _fixture(tmp_path, monkeypatch, "sqlite") + observed = tmp_path / "accepted-native-input" + monkeypatch.setenv("FAKE_OPERATION_PROMPT_FILE", str(observed)) + original_prompt = codex_operation_host._prompt + supplement = "完整输入☃" * (1024 * 1024) + monkeypatch.setattr(codex_operation_host, "_prompt", + lambda request: original_prompt(request) + "\n" + supplement) + code, payload = _cli([*arguments, "--execute"]) + assert code == 0 and payload["effects"]["host_invoked"], payload + assert turns.read_text().splitlines() == ["owned-app-server-thread"] + assert supplement in observed.read_text() diff --git a/tests/control_plane/test_contract_health.py b/tests/control_plane/test_contract_health.py index 12f14b76ad..3c64d5b061 100644 --- a/tests/control_plane/test_contract_health.py +++ b/tests/control_plane/test_contract_health.py @@ -15,6 +15,7 @@ from loopx.control_plane.quota.should_run import ( _apply_missing_state_health_guidance, ) +from loopx.control_plane.status.registry_health_projection import collect_global_registry_health from loopx.quota import build_quota_should_run @@ -22,6 +23,58 @@ GOAL_A_LONG = "goal-a-long" +@pytest.mark.parametrize("global_state", ["absent", "unreadable", "empty", "mirrored"]) +def test_local_declared_missing_state_is_health_fact_without_global_membership( + tmp_path: Path, global_state: str, +) -> None: + registry = tmp_path / "registry.json" + runtime = tmp_path / "runtime" + runtime.mkdir() + global_path = runtime / "registry.global.json" + payload = {"goals": [{"id": GOAL_A, "state_file": "missing.md"}]} + registry.write_text(json.dumps(payload)) + if global_state == "unreadable": + global_path.write_text("{broken") + elif global_state == "empty": + global_path.write_text(json.dumps({"goals": []})) + elif global_state == "mirrored": + global_path.write_text(json.dumps({"goals": [{ + "id": GOAL_A, "state_file": str(tmp_path / "missing.md"), + "source_registry": str(tmp_path / "missing-source.json"), + }]})) + health = collect_global_registry_health( + registry_path=registry, runtime_root=runtime, current_registry=payload, + ) + missing = [finding for finding in health["findings"] if finding["kind"] == "state_file_missing"] + assert len(missing) == 1 + assert missing[0]["path"] == str(tmp_path / "missing.md") + assert health["available"] is (global_state in {"empty", "mirrored"}) + assert goal_status_health_ok({"global_registry": health}, goal_id=GOAL_A, fallback=True) is False + assert goal_status_health_ok({"global_registry": health}, goal_id=GOAL_A_LONG, fallback=True) is True + if global_state == "unreadable": + assert health["ok"] is False + assert any(finding["kind"] == "global_registry_unreadable" for finding in health["findings"]) + if global_state == "mirrored": + assert "retire-global-goal" in missing[0]["recommended_action"] + assert any(finding["kind"] == "source_registry_missing" for finding in health["findings"]) + + +@pytest.mark.parametrize("declared", [False, True]) +def test_initial_state_setup_does_not_invent_local_missing_state( + tmp_path: Path, declared: bool, +) -> None: + state = tmp_path / "state.md" + state.write_text("# Initial Goal state\n") + payload = {"goals": [{"id": GOAL_A, **({"state_file": state.name} if declared else {})}]} + health = collect_global_registry_health( + registry_path=tmp_path / "registry.json", runtime_root=tmp_path / "runtime", + current_registry=payload, + ) + assert health["available"] is False and health["read_status"] == "missing" + assert health["findings"] == [] + assert goal_status_health_ok({"global_registry": health}, goal_id=GOAL_A, fallback=True) is True + + def _write_state(path: Path, *, malformed_user_todo: bool) -> None: user_section = "" if malformed_user_todo: diff --git a/tests/control_plane/test_first_party_host_runtime.py b/tests/control_plane/test_first_party_host_runtime.py index 746c95bf2a..95e6348ff6 100644 --- a/tests/control_plane/test_first_party_host_runtime.py +++ b/tests/control_plane/test_first_party_host_runtime.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + import threading import time from pathlib import Path @@ -14,6 +16,7 @@ from loopx.control_plane.goals.source_session_registry_state import guard_path from loopx.control_plane.projects.registry_codec import ( load_project_registry, + project_registry_transaction, source_session_registry_transaction, ) from loopx.file_lock import exclusive_cross_runtime_file_lock @@ -97,6 +100,73 @@ def test_legacy_profile_preserves_callbacks_and_bytes( assert registry.read_bytes() == before +def test_legacy_launch_stale_registry_requires_fresh_plan_without_rewriting_identity(tmp_path: Path) -> None: + registry = tmp_path / "registry.json" + with project_registry_transaction(registry, operation="fixture_create", create=lambda: { + "schema_version": "0.2", "goals": [{"id": "release"}], + }) as transaction: + transaction.commit(transaction.payload_copy()) + original = registry.read_bytes() + def plan(): + return FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, goal_id="release", planned_goal_ref=None, + ) + + admitted = plan() + with admitted.launch_lifetime(operation="fixture_launch"): + assert registry.read_bytes() == original + assert not admitted.enabled and admitted.planned_goal_ref is None + with project_registry_transaction(registry, operation="fixture_independent_configuration") as transaction: + payload = transaction.payload_copy() + payload["goals"].append({"id": "unrelated"}) + transaction.commit(payload) + changed = registry.read_bytes() + with pytest.raises(FirstPartyHostRuntimeRejected, match="stale_goal_authority"): + with admitted.launch_lifetime(operation="fixture_stale_launch"): + pytest.fail("stale registry cannot launch") + with plan().launch_lifetime(operation="fixture_fresh_launch"): + assert registry.read_bytes() == changed + assert registry.read_bytes() == changed + + +def test_legacy_launch_serializes_with_the_activation_registry_writer(tmp_path: Path) -> None: + registry = tmp_path / "registry.json" + with project_registry_transaction(registry, operation="fixture_create", create=lambda: { + "schema_version": "0.2", "goals": [{"id": "release"}], + }) as transaction: + transaction.commit(transaction.payload_copy()) + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, goal_id="release", planned_goal_ref=None, + ) + writer_started, stopped = threading.Event(), threading.Event() + + def stop_goal(): + writer_started.set() + with project_registry_transaction(registry, operation="fixture_stop") as transaction: + payload = transaction.payload_copy() + payload["goals"][0]["activation"] = { + "schema_version": "loopx_goal_activation_v1", "state": "stopped", + "updated_at": "2026-10-10T12:00:00Z", "reason": "Owner stopped this Goal", + } + transaction.commit(payload) + stopped.set() + + writer = threading.Thread(target=stop_goal) + try: + with admission.launch_lifetime(operation="fixture_launch"): + writer.start() + assert writer_started.wait(2) + assert not stopped.wait(0.1) + assert stopped.wait(2) + with pytest.raises(FirstPartyHostRuntimeRejected, match="goal_stopped"): + with admission.launch_lifetime(operation="fixture_stopped_launch"): + pytest.fail("stopped Goal cannot launch") + finally: + if writer.ident is not None: + writer.join(timeout=3) + assert not writer.is_alive() + + def test_source_profile_initializes_absent_state_with_exact_goal_ref( tmp_path: Path, ) -> None: @@ -126,6 +196,23 @@ def test_source_profile_initializes_absent_state_with_exact_goal_ref( } +def test_source_lifetime_does_not_nest_legacy_launch_quota_reader(tmp_path: Path) -> None: + registry = tmp_path / ".loopx" / "registry.json" + _write_source_registry(registry, INSTANCE_A) + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, goal_id="release", + planned_goal_ref={"goal_id": "release", "goal_instance_id": INSTANCE_A}, + ).with_launch_decision( + lambda _registry: pytest.fail("legacy quota admission nested inside source guard"), + ) + with admission.launch_lifetime(operation="fixture_exact_source_lifetime"): + assert admission.planned_goal_ref["goal_instance_id"] == INSTANCE_A + _replace_goal_instance(registry, INSTANCE_B) + with pytest.raises(FirstPartyHostRuntimeRejected, match="stale_goal_instance"): + with admission.launch_lifetime(operation="fixture_stale_exact_source_lifetime"): + pytest.fail("the exact source lifetime check must remain active") + + def test_source_profile_rejects_legacy_host_state_without_rewriting_it( tmp_path: Path, ) -> None: @@ -334,3 +421,52 @@ def commit_result() -> None: assert recreation_finished.is_set() is True goal = load_project_registry(registry)["goals"][0] assert goal["goal_instance_id"] == INSTANCE_B + + +@pytest.mark.parametrize("invalid_source", ["missing", "self", "cycle"]) +def test_legacy_mirror_launch_refuses_invalid_source_route_without_nested_lock( + tmp_path: Path, invalid_source: str, +) -> None: + mirror, source = tmp_path / "global.json", tmp_path / "source.json" + target = mirror if invalid_source == "self" else source + mirror.write_text(json.dumps({"schema_version": "0.2", "goals": [ + {"id": "release", "source_registry": str(target)}, + ]})) + if invalid_source == "cycle": + source.write_text(json.dumps({"schema_version": "0.2", "goals": [ + {"id": "release", "source_registry": str(mirror)}, + ]})) + before = mirror.read_bytes() + with pytest.raises(FirstPartyHostRuntimeRejected, match="goal_authority_unavailable"): + FirstPartyHostGoalAdmission.for_plan( + registry_path=mirror, goal_id="release", planned_goal_ref=None, + ) + assert mirror.read_bytes() == before + with project_registry_transaction(mirror, operation="fixture_route_lock_released"): + pass + + +@pytest.mark.parametrize("padding", ["", " "]) +def test_legacy_mirror_source_witness_requires_fresh_plan_after_independent_change(tmp_path: Path, padding: str) -> None: + mirror, source = tmp_path / "global.json", tmp_path / "source.json" + source.write_text(json.dumps({"schema_version": "0.2", "goals": [{"id": "release"}]})) + mirror.write_text(json.dumps({"schema_version": "0.2", "goals": [ + {"id": "release", "source_registry": padding + str(source) + padding}, + ]})) + def plan(): + return FirstPartyHostGoalAdmission.for_plan( + registry_path=mirror, goal_id="release", planned_goal_ref=None, + ) + admitted = plan() + before = mirror.read_bytes() + with admitted.launch_lifetime(operation="fixture_initial_mirror_launch"): + assert not admitted.enabled and admitted.planned_goal_ref is None + with project_registry_transaction(source, operation="fixture_unrelated_source_edit") as transaction: + payload = transaction.payload_copy() + payload["goals"].append({"id": "unrelated"}) + transaction.commit(payload) + with pytest.raises(FirstPartyHostRuntimeRejected, match="stale_goal_authority"): + with admitted.launch_lifetime(operation="fixture_stale_source_launch"): + pytest.fail("a mirror cannot override a changed source witness") + with plan().launch_lifetime(operation="fixture_fresh_source_launch"): + assert mirror.read_bytes() == before diff --git a/tests/control_plane/test_host_current_admission.py b/tests/control_plane/test_host_current_admission.py new file mode 100644 index 0000000000..b984789eaa --- /dev/null +++ b/tests/control_plane/test_host_current_admission.py @@ -0,0 +1,96 @@ +"""Current source quota and health must still allow the actual native launch.""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.configure_goal import configure_goal +from loopx.control_plane.turn_driver import codex_cli, codex_operation_host +from loopx.global_registry import sync_project_registry_to_global +from tests.control_plane.test_codex_operation_launch import _bindings, _cli, _fixture +from tests.test_loopx_turn_codex_cli import _fake_codex + + +def _quota(registry: Path, runtime: Path, project: Path) -> dict: + _, payload = _cli([ + "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", + "quota", "should-run", "--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture", + "--host-surface", "generic_cli", "--scheduler-owner", "outer_controller", + "--execution-mode", "isolated_headless", "--scan-root", str(project), + ]) + return payload + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("refusal", ["source_compute_zero", "missing_state"]) +@pytest.mark.parametrize("host", ["exec", "operation"]) +def test_current_source_admission_preempts_stale_mirror_native_launch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, refusal: str, host: str, +) -> None: + runtime, source, native, model, arguments = _fixture(tmp_path, monkeypatch, provider) + project = source.parent.parent + synchronized = sync_project_registry_to_global( + registry_path=source, runtime_root_override=str(runtime), goal_id="loopx-turn-fixture", + ) + assert synchronized["ok"], synchronized + mirror = runtime / "registry.global.json" + before_mirror = mirror.read_bytes() + source_payload = json.loads(source.read_text()) + state = project / source_payload["goals"][0]["state_file"] + original_state = state.read_bytes() + if host == "exec": + executable, model = _fake_codex(tmp_path) + native = tmp_path / "native-process" + executable.write_text(executable.read_text().replace( + "args = sys.argv[1:]", f"pathlib.Path({str(native)!r}).write_text('native')\nargs = sys.argv[1:]", 1, + )) + monkeypatch.setenv("FAKE_CODEX_LOG", str(model)) + arguments.remove("--codex-operation-tools") + arguments[arguments.index("--codex-bin") + 1] = str(executable) + arguments[arguments.index("--registry") + 1] = str(mirror) + observations = [] + if refusal == "source_compute_zero": + configured = configure_goal(registry_path=source, goal_id="loopx-turn-fixture", quota_compute=0, execute=True) + assert configured["written"], configured + current = _quota(source, runtime, project) + assert current["should_run"] is False and "compute_quota_zero" in json.dumps(current), current + assert _quota(mirror, runtime, project)["should_run"] is True + observations.append(current) + else: + def lose_state_after_selection(): + state.rename(state.with_suffix(".saved")) + for registry in (source, mirror): + current = _quota(registry, runtime, project) + assert current["should_run"] is False and current["status_health_ok"] is False, current + assert current["state"] == "blocked_health", current + observations.append(current) + if host == "exec": + original_prompt = codex_cli._prompt + + def lose_state_before_prompt(request): + lose_state_after_selection() + return original_prompt(request) + + monkeypatch.setattr(codex_cli, "_prompt", lose_state_before_prompt) + else: + original_start = codex_operation_host.CodexChatAgentSession.start + + def lose_state_before_start(**kwargs): + lose_state_after_selection() + return original_start(**kwargs) + + monkeypatch.setattr(codex_operation_host.CodexChatAgentSession, "start", lose_state_before_start) + before_bindings = _bindings(runtime) + code, payload = _cli([*arguments, "--execute"]) + assert observations and mirror.read_bytes() == before_mirror + assert not native.exists() and not model.exists(), payload + assert code == 1 and "goal_launch_not_allowed" in json.dumps(payload), payload + assert _bindings(runtime) == before_bindings + if refusal == "source_compute_zero": + assert state.read_bytes() == original_state + else: + assert not state.exists() + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() diff --git a/tests/control_plane/test_host_plan_read_only.py b/tests/control_plane/test_host_plan_read_only.py new file mode 100644 index 0000000000..4b1bbaca5d --- /dev/null +++ b/tests/control_plane/test_host_plan_read_only.py @@ -0,0 +1,100 @@ +"""Read-only planning cannot create launch locks or bypass current authority.""" +from __future__ import annotations + +import contextlib +import io +import json +from pathlib import Path +import sys + +import pytest + +from loopx.cli import main as cli_main +from loopx.control_plane.goals.first_party_host_admission import ( + FirstPartyHostGoalAdmission, + FirstPartyHostRuntimeRejected, + capture_first_party_host_goal_ref, +) +from tests.control_plane.test_first_party_host_runtime import INSTANCE_A, _write_source_registry +from tests.test_loopx_turn_driver import _write_live_fixture + + +def _snapshot(root: Path) -> dict[str, bytes | None]: + # Retain every path, including empty directories and registry lock files. + return { + str(path.relative_to(root)): path.read_bytes() if path.is_file() else None + for path in root.rglob("*") + } + + +def _legacy_registry(root: Path) -> Path: + registry = root / "registry.json" + registry.write_text(json.dumps({ + "schema_version": "0.2", "goals": [{"id": "release", "status": "active"}], + }) + "\n", encoding="utf-8") + return registry + + +def test_read_only_plan_detects_a_source_profile_published_after_capture(tmp_path: Path) -> None: + registry = _legacy_registry(tmp_path) + captured = capture_first_party_host_goal_ref(registry_path=registry, goal_id="release") + assert captured is None + # Publish a valid source profile between the CLI's two authority reads. + # The existing source owner constructs the replacement before publication. + replacement = tmp_path / "replacement.json" + _write_source_registry(replacement, INSTANCE_A) + replacement.replace(registry) + before = _snapshot(tmp_path) + + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, goal_id="release", planned_goal_ref=captured, + capture_launch_authority=False, + ) + + assert admission.enabled is True + assert admission.planned_goal_ref is None + assert _snapshot(tmp_path) == before + with pytest.raises(FirstPartyHostRuntimeRejected) as rejected: + admission.require_current() + assert rejected.value.code == "goal_instance_id_missing" + + +def test_read_only_legacy_admission_cannot_grant_host_launch(tmp_path: Path) -> None: + registry = _legacy_registry(tmp_path) + before = _snapshot(tmp_path) + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry, goal_id="release", planned_goal_ref=None, + capture_launch_authority=False, + ) + assert admission.enabled is False + assert admission.legacy_registry_witness is None + assert _snapshot(tmp_path) == before + + with pytest.raises(FirstPartyHostRuntimeRejected) as rejected: + with admission.launch_lifetime(operation="read_only_plan_cannot_launch"): + pytest.fail("a read-only plan without its original witness cannot launch a Host") + assert rejected.value.code == "goal_authority_unavailable" + + +def test_run_once_preview_preserves_every_fixture_path_and_file(tmp_path: Path) -> None: + project, runtime, registry = _write_live_fixture(tmp_path) + before = _snapshot(tmp_path) + output = io.StringIO() + with contextlib.redirect_stdout(output): + exit_code = cli_main([ + "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", + "turn", "run-once", "--host", "generic-cli", "--goal-id", "loopx-turn-fixture", + "--agent-id", "codex-fixture", "--project", str(project), "--scan-root", str(project), + "--host-command-json", json.dumps([ + sys.executable, "-c", "raise AssertionError('preview invoked a Host')", + ]), + "--validation-command-json", json.dumps([ + sys.executable, "-c", "raise AssertionError('preview ran validation')", + ]), + ]) + + payload = json.loads(output.getvalue()) + assert exit_code == 0, payload + assert payload["effects"]["state_written"] is False + assert payload["effects"]["host_invoked"] is False + assert _snapshot(tmp_path) == before diff --git a/tests/control_plane/test_host_process.py b/tests/control_plane/test_host_process.py index c88e88ae8e..6e44644e77 100644 --- a/tests/control_plane/test_host_process.py +++ b/tests/control_plane/test_host_process.py @@ -8,6 +8,7 @@ import subprocess import sys import time +from contextlib import contextmanager from pathlib import Path import pytest @@ -22,6 +23,56 @@ from tests.control_plane.host_process_fixture import COUNTER_PROCESS_SOURCE +def test_launch_guard_releases_before_running_host_output(tmp_path: Path) -> None: + held = [] + observed = [] + + @contextmanager + def admission(): + held.append(True) + try: + yield + finally: + held.pop() + + def output(text: str) -> None: + # A running Host must not hold the owner lock and delay a user stop. + assert not held + observed.append(text) + + result = run_host_process( + [sys.executable, "-c", "print('current work')"], project=tmp_path, + input_text="", timeout_seconds=5, launch_guard=admission, on_stdout=output, + ) + assert result["returncode"] == 0 and observed and not held + failed = run_host_process( + [str(tmp_path / "absent-host")], project=tmp_path, + input_text="", timeout_seconds=5, launch_guard=admission, + ) + assert failed["outcome"] == "spawn_failed" and not held + + +def test_rejected_launch_guard_keeps_nonexecution_evidence(tmp_path: Path, monkeypatch) -> None: + from loopx.control_plane.turn_driver.host_process_transport import prepare_host_process_record + + marker, record = tmp_path / "work", tmp_path / "host.json" + prepare_host_process_record(record) + before = record.read_bytes() + monkeypatch.setenv(HOST_PROCESS_RECORD_ENV, str(record)) + + @contextmanager + def stopped(): + raise ValueError("Goal was stopped after planning") + yield + + with pytest.raises(ValueError, match="stopped after planning"): + run_host_process( + [sys.executable, "-c", "from pathlib import Path;Path('work').touch()"], + project=tmp_path, input_text="", timeout_seconds=5, launch_guard=stopped, + ) + assert not marker.exists() and record.read_bytes() == before + + def test_host_output_lines_bound_storage_and_use_lf() -> None: rows: list[str] = [] lines = HostOutputLines(rows.append, max_chars=20) diff --git a/tests/control_plane/test_host_research_recovery_cli.py b/tests/control_plane/test_host_research_recovery_cli.py new file mode 100644 index 0000000000..8ef7fe9325 --- /dev/null +++ b/tests/control_plane/test_host_research_recovery_cli.py @@ -0,0 +1,260 @@ +"""Original research results recover before a new review reaches a real Host. + +The model is a deterministic local child. CLI planning, canonical providers, +the original journal, writeback and quota remain production implementations. +""" +from __future__ import annotations + +import argparse +import contextlib +import io +import json +from pathlib import Path +import shlex +import subprocess +import sys + +import pytest + +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from loopx.cli import main as cli_main +from loopx.cli_commands import turn_run_once as turn_command +from loopx.goal_mode_mcp import GoalModeMCPConfig, GoalModeMCPControlPlane +from tests import test_loopx_turn_driver as turn_fixture +from tests.test_external_scheduler_worker import worker + +GOAL = "loopx-turn-fixture" +AGENT = "codex-fixture" +TODO = "todo_fixture0001" +ROOT = Path(__file__).resolve().parents[2] + + +def _cli(registry, runtime, *args, code=0): + completed = subprocess.run([sys.executable, "-m", "loopx.cli", "--registry", str(registry), + "--runtime-root", str(runtime), "--format", "json", *args], + cwd=ROOT, capture_output=True, text=True, timeout=90) + result = json.loads(completed.stdout) + assert completed.returncode == code, json.dumps(result, ensure_ascii=False) + return result + + +def _control(registry, runtime): + control = GoalModeMCPControlPlane( + GoalModeMCPConfig(server_name="research-recovery", runtime_profile="generic_cli", + legacy_host_surface="claude_code"), + lambda: {"goal_id": GOAL, "agent_id": AGENT, "registry": str(registry)}, + ) + control.command_prefix = lambda: [sys.executable, "-m", "loopx.cli", "--runtime-root", str(runtime)] + return control + + +def _in_process(argv, code=0): + output = io.StringIO() + with contextlib.redirect_stdout(output): + observed = cli_main(argv) + result = json.loads(output.getvalue()) + assert observed == code, json.dumps(result, ensure_ascii=False) + return result + + +def _spends(index): + return [row for line in index.read_text().splitlines() + if (row := json.loads(line)).get("classification") == "quota_slot_spent"] + + +def _worker_args(project, runtime, registry, script, request): + launcher = project / "fixture-loopx" + launcher.write_text("#!/bin/sh\nexec " + shlex.join([sys.executable, "-m", "loopx.cli"]) + ' "$@"\n') + launcher.chmod(0o700) + return argparse.Namespace(cli_bin=str(launcher), registry=str(registry), runtime_root=str(runtime), + runtime_profile="generic_cli", goal_id=GOAL, agent_id=AGENT, + state_file=str(project / "worker-state.json"), once=True, + wake_cmd=shlex.join([sys.executable, str(script), str(request)]), + error_backoff_seconds=5, quota_timeout_seconds=90, wake_timeout_seconds=120) + + +_REVIEW_HOST = r'''import json,sys +from hashlib import sha256 +from pathlib import Path +import subprocess +request=json.loads(Path(sys.argv[1]).read_text()) +def cli(*args): + result=subprocess.run([sys.executable,"-m","loopx.cli","--registry",request["registry"], + "--runtime-root",request["runtime"],"--format","json",*args],capture_output=True,text=True,timeout=60) + payload=json.loads(result.stdout) + if result.returncode: + Path(request["result"]).write_text(json.dumps({"failed_command":args,"result":payload})) + assert result.returncode==0,payload + return payload +guard=cli("quota","should-run","--runtime-profile","generic_cli","--goal-id",request["goal"], + "--agent-id",request["agent"],"--turn-instance-id",request["review_turn"]) +assert guard["effective_action"]=="autonomous_replan_required",guard +identity=guard["heartbeat_receipt"]["settlement_identity"] +assert identity["replan_obligation_id"]==request["obligation"],guard +with Path(request["counts"]).open("a") as stream: + stream.write(json.dumps({"work":"R2","D":1,"P":1})+"\n") +original=Path(request["original"]) +assert original.read_text()=="completed" +artifact=Path(request["artifact"]) +artifact.write_text(json.dumps({"review":"R2","original_sha256":sha256(original.read_bytes()).hexdigest(), + "adopted_original":True})+"\n") +evidence="fixture:review:"+sha256(artifact.read_bytes()).hexdigest() +vision={"schema_version":"goal_vision_replan_contract_v0","state":"no_followup", + "vision_patch":{"vision_summary":"Complete and independently read back the finite original probe.", + "acceptance_summary":"The original artifact was verified and its result was recovered."}, + "path_delta":{"schema_version":"goal_path_delta_v0","outcome":"stop", + "prior_assumption":"The completed original probe still required a current review.", + "observed_reality":"The existing original artifact passed the independently repeated readback.", + "retained":["The original artifact and its operation identity."],"evidence_refs":[evidence]}} +vision_path=artifact.with_suffix(".vision.json") +vision_path.write_text(json.dumps(vision)) +binding=["--goal-id",request["goal"],"--agent-id",request["agent"], + "--turn-instance-id",identity["turn_instance_id"],"--replan-obligation-id",identity["replan_obligation_id"]] +refreshed=cli("refresh-state",*binding,"--classification","bounded_research_review", + "--delivery-batch-scale","implementation","--delivery-outcome","outcome_progress", + "--agent-vision-json",str(vision_path),"--progress-result-class","no_followup", + "--progress-coverage-scope-id","original-probe","--progress-coverage-complete", + "--progress-evidence-id",evidence,"--no-global-sync","--suppress-external-sinks") +assert refreshed["settlement_progress"]["state"]=="spend_required",refreshed +spent=cli("quota","spend-slot",*binding,"--slots","1","--source","heartbeat","--execute") +assert spent["settlement_progress"]["state"]=="settled",spent +Path(request["result"]).write_text(json.dumps({"identity":identity,"refreshed":refreshed,"spent":spent})) +''' + + +_HEARTBEAT_HOST = r'''import json,sys +from hashlib import sha256 +from pathlib import Path +from loopx.goal_mode_mcp import GoalModeMCPConfig,GoalModeMCPControlPlane +request=json.loads(Path(sys.argv[1]).read_text()) +control=GoalModeMCPControlPlane(GoalModeMCPConfig(server_name="original-heartbeat", + runtime_profile="generic_cli",legacy_host_surface="claude_code"), + lambda:{"goal_id":request["goal"],"agent_id":request["agent"],"registry":request["registry"]}) +control.command_prefix=lambda:[sys.executable,"-m","loopx.cli","--runtime-root",request["runtime"]] +original_run_cli=control.run_cli +def observed_cli(*args,**kwargs): + raw=original_run_cli(*args,**kwargs) + with Path(request["result"]+".trace.jsonl").open("a") as stream: + stream.write(json.dumps({"args":args,"result":json.loads(raw)})+"\n") + return raw +control.run_cli=observed_cli +guard=json.loads(control.should_run()) +assert guard["should_run"] and guard["selected_todo"]["todo_id"]==request["todo"],guard +with Path("host-counts.jsonl").open("a") as stream: + stream.write(json.dumps({"work":"O3","D":1,"P":1})+"\n") +artifact=Path("completion-artifact.txt") +artifact.write_text("completed") +evidence="fixture:original:"+sha256(artifact.read_bytes()).hexdigest() +completed=json.loads(control.complete_task(request["todo"],request["agent"],evidence,no_follow_up=True)) +Path(request["result"]).write_text(json.dumps({"completion":completed,"evidence":evidence})) +assert completed["ok"] is True,completed +# The provider accepted the result. Only the outer Host response is lost. +raise SystemExit(1) +''' + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("mode", ["governed", "heartbeat"]) +def test_original_journal_recovers_before_review_host_once(tmp_path, monkeypatch, provider, mode): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = turn_fixture._write_live_fixture( + tmp_path, provider=provider, todo_metadata_extra="no_followup=true" if mode == "governed" else "") + monkeypatch.setenv("PYTHONPATH", str(ROOT)) + state = project / f".codex/goals/{GOAL}/ACTIVE_GOAL_STATE.md" + state.write_text(state.read_text() + "\n## Objective\nComplete and independently read back the finite original probe.\n\n## User Todo / Owner Review Reading Queue\n") + _cli(registry, runtime, "configure-goal", "--goal-id", GOAL, + "--execution-replan-after-todos", "5", "--execute") + index = runtime / "goals" / GOAL / "runs/index.jsonl" + index.parent.mkdir(parents=True, exist_ok=True) + # The nineteenth prior work result is fixture setup, before O3 admission. + evidence = project / "prior-results.json" + evidence.write_text(json.dumps({"accepted_prior_results": 19})) + report = project / "prior-results.md" + report.write_text("Nineteen prior bounded work results were accepted.\n") + index.write_text("".join(json.dumps({"goal_id": GOAL, "agent_id": AGENT, + "classification": "evidence_validated", "delivery_outcome": "outcome_progress", + "turn_instance_id": f"prior-work-{i}", "generated_at": f"2026-08-01T00:{i:02d}:00Z", + "json_path": str(evidence), "markdown_path": str(report)}) + "\n" for i in range(19))) + before = _cli(registry, runtime, "quota", "should-run", "--runtime-profile", "generic_cli", + "--goal-id", GOAL, "--agent-id", AGENT) + assert before["selected_todo"]["todo_id"] == TODO + assert before.get("autonomous_replan_obligation") is None + workspace = project / "host-workspace" + workspace.mkdir() + host, validate = turn_fixture._completion_host_and_validation_scripts() + host = host.replace('request = json.load(sys.stdin)', + 'request = json.load(sys.stdin)\nwith pathlib.Path("host-counts.jsonl").open("a") as stream:\n stream.write(json.dumps({"work":"O3","D":1,"P":1})+"\\n")') + argv = turn_fixture._turn_run_once_completion_argv(workspace, runtime, registry, host, validate) + real_append = turn_command.append_cli_rollout_event + + def lost_ack(payload, **kwargs): + if kwargs.get("event_kind") == "quota_spend": + raise OSError("fixture lost original completion ACK") + return real_append(payload, **kwargs) + + if mode == "governed": + with monkeypatch.context() as fault: + fault.setattr(turn_command, "append_cli_rollout_event", lost_ack) + first = _in_process(argv, code=1) + assert first["error"] == "fixture lost original completion ACK" + journal = turn_fixture._turn_journal(runtime) + turn_key = journal["turn_key"] + + def recover_original(): + replay = _in_process([*argv[:-1], "--resume-turn-key", turn_key, "--execute"]) + assert replay["status"] == "committed" and replay["effects"]["host_invoked"] is False + else: + original_host, original_request, original_result = ( + project / "original-host.py", project / "original-request.json", project / "original-result.json") + original_host.write_text(_HEARTBEAT_HOST) + original_request.write_text(json.dumps({"registry": str(registry), "runtime": str(runtime), + "goal": GOAL, "agent": AGENT, "todo": TODO, "result": str(original_result)})) + process = subprocess.run([sys.executable, str(original_host), str(original_request)], + cwd=workspace, capture_output=True, text=True, timeout=120) + assert process.returncode == 1 + original = json.loads(original_result.read_text()) + assert original["completion"]["ok"] is True, json.dumps(original, ensure_ascii=False) + control = _control(registry, runtime) + + def recover_original(): + replay = json.loads(control.complete_task(TODO, AGENT, original["evidence"], no_follow_up=True)) + assert replay["ok"] is True, replay + original_bytes = (workspace / "completion-artifact.txt").read_bytes() + counts = workspace / "host-counts.jsonl" + assert len(counts.read_text().splitlines()) == 1 + assert len(_spends(index)) == 1 + current = _cli(registry, runtime, "quota", "should-run", "--runtime-profile", "generic_cli", + "--goal-id", GOAL, "--agent-id", AGENT) + assert current["effective_action"] == "autonomous_replan_required", current + obligation = current["autonomous_replan_obligation"]["obligation_id"] + # An old unbound write is not a substitute for resuming the original journal. + stale = _cli(registry, runtime, "refresh-state", "--goal-id", GOAL, "--agent-id", AGENT, + "--todo-id", TODO, "--classification", "stale_original_submission", + "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", + "--no-global-sync", "--suppress-external-sinks", code=1) + assert not stale.get("appended") + recover_original() + assert len(_spends(index)) == 1 + assert (workspace / "completion-artifact.txt").read_bytes() == original_bytes + assert len(counts.read_text().splitlines()) == 1 + + review = project / "review-host.py" + review.write_text(_REVIEW_HOST) + request = project / "review-request.json" + artifact, result = project / "review.json", project / "review-result.json" + request.write_text(json.dumps({"registry": str(registry), "runtime": str(runtime), + "goal": GOAL, "agent": AGENT, "obligation": obligation, "counts": str(counts), + "review_turn": "fixture-current-review-after-original", + "original": str(workspace / "completion-artifact.txt"), "artifact": str(artifact), "result": str(result)})) + args = _worker_args(project, runtime, registry, review, request) + assert worker.run_worker(args) == 0, result.read_text() if result.exists() else "Host failed before writeback" + assert json.loads(artifact.read_text())["adopted_original"] is True + adopted = json.loads(result.read_text()) + assert adopted["identity"]["replan_obligation_id"] == obligation + assert len(_spends(index)) == 2 + for _ in range(2): + recover_original() + assert worker.run_worker(args) == 0 + assert [json.loads(row)["work"] for row in counts.read_text().splitlines()] == ["O3", "R2"] + assert len(_spends(index)) == 2 + assert (workspace / "completion-artifact.txt").read_bytes() == original_bytes diff --git a/tests/control_plane/test_initial_planning_host_cli.py b/tests/control_plane/test_initial_planning_host_cli.py new file mode 100644 index 0000000000..940f8bb305 --- /dev/null +++ b/tests/control_plane/test_initial_planning_host_cli.py @@ -0,0 +1,136 @@ +"""Initial Goal planning and unbound read-only Host paths stay distinct.""" +from __future__ import annotations + +import json +import pytest + +from tests.control_plane.test_replan_successor_guard_reentry import AGENT, GOAL, _fixture +from tests.control_plane.test_host_research_recovery_cli import ROOT, _spends, _worker_args +from tests.test_external_scheduler_worker import worker + + +_PLANNING_HOST = r'''import json,sys,uuid +from hashlib import sha256 +from pathlib import Path +import subprocess +from loopx.goal_mode_mcp import GoalModeMCPConfig,GoalModeMCPControlPlane +request=json.loads(Path(sys.argv[1]).read_text()) +def failed(kind,error,tb): + import traceback + Path(request["result"]).write_text(json.dumps({"error_kind":kind.__name__,"error":str(error), + "traceback":traceback.format_exception(kind,error,tb,limit=8)})) + sys.__excepthook__(kind,error,tb) +sys.excepthook=failed +def cli(*args): + result=subprocess.run([sys.executable,"-m","loopx.cli","--registry",request["registry"], + "--runtime-root",request["runtime"],"--format","json",*args],capture_output=True,text=True,timeout=60) + payload=json.loads(result.stdout) + if result.returncode: + Path(request["result"]).write_text(json.dumps({"failed_command":args,"result":payload})) + assert result.returncode==0,payload + return payload +turn="initial-host-"+uuid.uuid4().hex +common=["--goal-id",request["goal"],"--agent-id",request["agent"]] +guard=cli("quota","should-run","--runtime-profile","generic_cli",*common) +assert guard["should_run"] is True,guard +todo=guard.get("selected_todo") +role="work" if todo else "plan" +with Path(request["counts"]).open("a") as stream: + stream.write(json.dumps({"role":role,"D":1,"P":1})+"\n") +if role=="plan": + admitted=cli("quota","should-run","--runtime-profile","generic_cli",*common,"--turn-instance-id",turn) + assert admitted["should_run"] and admitted.get("selected_todo") is None,admitted + artifact=Path(request["plan"]) + artifact.write_text(json.dumps({"objective":guard["goal_id"],"probe":"Verify the retained two-number sum.", + "inputs":[2,3],"expected":5})+"\n") + added=cli("todo","add","--goal-id",request["goal"],"--role","agent","--claimed-by",request["agent"], + "--text","Verify the retained two-number sum from the accepted initial plan.", + "--task-class","advancement_task","--action-kind","validate","--operation-id","accepted-initial-plan") + selected=cli("quota","should-run","--runtime-profile","generic_cli",*common,"--turn-instance-id",turn) + assert selected["selected_todo"]["todo_id"]==added["todo_id"],selected + binding=[*common,"--turn-instance-id",turn,"--todo-id",added["todo_id"]] + refreshed=cli("refresh-state",*binding,"--classification","initial_planning_completed", + "--delivery-batch-scale","implementation","--delivery-outcome","outcome_progress", + "--no-global-sync","--suppress-external-sinks") + spent=cli("quota","spend-slot",*binding,"--slots","1","--source","heartbeat","--execute") + assert spent["appended"] is True,spent + Path(request["result"]).write_text(json.dumps({"planning_turn":turn,"todo_id":added["todo_id"]})) +else: + plan=json.loads(Path(request["plan"]).read_text()) + assert "accepted initial plan" in todo["text"],todo + result={"sum":sum(plan["inputs"]),"expected":plan["expected"]} + assert result["sum"]==result["expected"] + artifact=Path(request["work"]) + artifact.write_text(json.dumps(result)+"\n") + evidence="fixture:initial-plan-adoption:"+sha256(artifact.read_bytes()).hexdigest() + vision={"schema_version":"goal_vision_replan_contract_v0","state":"no_followup", + "vision_patch":{"vision_summary":"Verify the finite two-number sum.", + "acceptance_summary":"The plan was adopted and its explicit sum was verified."}, + "path_delta":{"schema_version":"goal_path_delta_v0","outcome":"no_change", + "prior_assumption":"The first planned probe remained unexecuted.", + "observed_reality":"The explicitly planned sum matched its declared expected value.", + "stopped":["No additional work belongs to this finite probe."],"evidence_refs":[evidence]}} + control=GoalModeMCPControlPlane(GoalModeMCPConfig(server_name="initial-plan",runtime_profile="generic_cli", + legacy_host_surface="claude_code"),lambda:{"goal_id":request["goal"],"agent_id":request["agent"],"registry":request["registry"]}) + control.command_prefix=lambda:[sys.executable,"-m","loopx.cli","--runtime-root",request["runtime"]] + completed=json.loads(control.complete_task(todo["todo_id"],request["agent"],evidence,no_follow_up=True,agent_vision=vision)) + Path(request["result"]).write_text(json.dumps({"completion":completed})) + assert completed["ok"] is True,completed +''' + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_fresh_goal_without_todos_can_admit_its_first_planning(tmp_path, monkeypatch, provider): + call, runtime, index = _fixture(tmp_path, monkeypatch, provider, future_monitor=False) + # Remove the helper's prior history before the new Goal's first observation. + # The accepted objective and empty canonical Todo source remain real. + index.write_text("") + packet = call("quota", "should-run", "--runtime-profile", "generic_cli", + "--goal-id", GOAL, "--agent-id", AGENT) + assert packet.get("selected_todo") is None + assert packet["should_run"] is True, json.dumps(packet) + project, registry = tmp_path / "project", tmp_path / "registry.json" + # The synthetic Host produces its plan and work in this Goal project. + # Keep the real delivery-workspace guard independent of pytest's checkout. + monkeypatch.chdir(project) + monkeypatch.setenv("PYTHONPATH", str(ROOT)) + host, request, result = project / "planning-host.py", project / "request.json", project / "result.json" + counts, plan, work = project / "counts.jsonl", project / "plan.json", project / "work.json" + host.write_text(_PLANNING_HOST) + request.write_text(json.dumps({"registry": str(registry), "runtime": str(runtime), "goal": GOAL, + "agent": AGENT, "counts": str(counts), "plan": str(plan), "work": str(work), "result": str(result)})) + args = _worker_args(project, runtime, registry, host, request) + args.goal_id, args.agent_id = GOAL, AGENT + # Observation never adopts the otherwise runnable first planning request. + wake_cmd, args.wake_cmd = args.wake_cmd, None + assert worker.run_worker(args) == 0 + assert not counts.exists() and not plan.exists() + args.wake_cmd = wake_cmd + assert worker.run_worker(args) == 0, result.read_text() if result.exists() else "Host failed before planning" + initial = json.loads(result.read_text()) + todo = call("todo", "list", "--goal-id", GOAL, "--todo-id", initial["todo_id"]) + assert todo["todo"]["status"] == "open" + assert len(_spends(index)) == 1 + original = call("quota", "spend-slot", "--goal-id", GOAL, "--agent-id", AGENT, + "--turn-instance-id", initial["planning_turn"], "--todo-id", initial["todo_id"], + "--slots", "1", "--source", "heartbeat", "--execute") + assert original["appended"] is False and original["idempotent_replay"] is True + assert len(_spends(index)) == 1 + # A fresh worker now consumes the concrete planned task, not initialization again. + assert worker.run_worker(args) == 0, result.read_text() + assert json.loads(work.read_text()) == {"sum": 5, "expected": 5} + assert worker.run_worker(args) == 0 + assert [json.loads(line)["role"] for line in counts.read_text().splitlines()] == ["plan", "work"] + assert call("todo", "list", "--goal-id", GOAL, "--todo-id", initial["todo_id"])["todo"]["status"] == "done" + assert len(_spends(index)) == 2 + + +def test_unbound_chat_adapter_does_not_start_goal_research(monkeypatch): + from loopx.goal_mode_mcp import GoalModeMCPConfig, GoalModeMCPControlPlane + control = GoalModeMCPControlPlane( + GoalModeMCPConfig(server_name="ordinary-chat", runtime_profile="generic_cli", legacy_host_surface="claude_code"), + lambda: {}, + ) + monkeypatch.setattr(control, "run_cli", lambda *_a, **_kw: pytest.fail("unbound chat must not execute Goal commands")) + assert json.loads(control.should_run())["ok"] is False + assert json.loads(control.complete_task("unbound-task", "unbound-agent", "no Goal authority"))["ok"] is False diff --git a/tests/control_plane/test_phase_probe_continuation.py b/tests/control_plane/test_phase_probe_continuation.py new file mode 100644 index 0000000000..e69fb94765 --- /dev/null +++ b/tests/control_plane/test_phase_probe_continuation.py @@ -0,0 +1,312 @@ +"""A separately authorized phase is consumed by the resumed native Host. + +Only inference and candidate grading are deterministic fixtures. Native +controller activation, CLI/Host session reuse, canonical Todo acceptance and +quota settlement use the existing owners, with no real model requests. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import sys +import time +from types import SimpleNamespace + +import pytest + +from benchmark.edgebench import checkpoint_controller as transport +from benchmark.runtime import worker as benchmark_worker +from benchmark.tests import test_checkpoint_controller as checkpoint_fixture +from benchmark.tests.test_shared_codex_runtime import worker_env +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests import test_loopx_turn_driver as fixture +from loopx.control_plane.goals.activation_service import set_goal_activation_state +from tests.control_plane.test_host_research_recovery_cli import AGENT, GOAL, ROOT, TODO, _cli, _in_process, _spends + + +controllers = checkpoint_fixture.controllers +register = checkpoint_fixture.register +submit = checkpoint_fixture.submit + + +_PHASE_HOST = r'''import json,os,re,sys,time +from pathlib import Path +args=sys.argv[1:] +prompt=sys.stdin.read() +request=json.loads(prompt.rsplit("Turn request:\n",1)[1]) +selected=request["turn_envelope"]["action"]["selected_todo"] +text=request["turn_envelope"]["action"]["recommended_action"] +phase=re.search(r"Phase (S[12])",text).group(1) +inputs=json.loads(Path("retained-inputs.json").read_text()) +history=Path("native-history.jsonl") +if phase=="S2": + assert "resume" in args and "retained-phase-session" in args,args + old=json.loads(history.read_text().splitlines()[0]) + assert old["phase"]=="S1" and old["old_deadline"]float(os.environ["FIXTURE_HOST_NOW"]) + assert "sum of squares" in text,text + value=sum(number*number for number in inputs) +else: + assert "sum of retained inputs" in text,text + value=sum(inputs) +artifact={"phase":phase,"inputs":inputs,"value":value,"consumed_todo_id":selected["todo_id"]} +Path("probe-"+phase+".json").write_text(json.dumps(artifact)+"\n") +with history.open("a") as stream: + stream.write(json.dumps({"phase":phase,"old_deadline":int(os.environ["FIXTURE_S1_DEADLINE"]),"value":value})+"\n") +with Path("model-counts.jsonl").open("a") as stream: + stream.write(json.dumps({"phase":phase,"D":1,"P":1})+"\n") +print(json.dumps({"type":"thread.started","thread_id":"retained-phase-session"}),flush=True) +Path(args[args.index("--output-last-message")+1]).write_text(json.dumps({ + "schema_version":"loopx_turn_result_v0","turn_key":request["turn_key"], + "result_kind":"validated_completion","completed_phases":["host_execute","typed_result"], + "classification":"phase_probe_completed","recommended_action":"Read the current Goal after accepting this probe.", + "next_action":"Use current independently authorized work.","delivery_batch_scale":"implementation", + "delivery_outcome":"outcome_progress","vision_unchanged_reason":"The finite authorized probe remains the accepted scope.", + "summary":"The current phase probe executed on the retained inputs."})) +''' + + +_HEARTBEAT_PHASE_HOST = r'''import json,os,re,subprocess,sys +from hashlib import sha256 +from pathlib import Path +args=sys.argv[1:] +prompt=sys.stdin.read() +assert os.environ["LOOPX_GOAL_ID"] in prompt,prompt +def cli(*arguments): + completed=subprocess.run([os.environ["LOOPX_CLI"],"--registry",os.environ["LOOPX_REGISTRY"], + "--runtime-root",os.environ["LOOPX_RUNTIME_ROOT"],"--format","json",*arguments], + capture_output=True,text=True,timeout=90) + result=json.loads(completed.stdout) + if completed.returncode: + Path("phase-host-error.json").write_text(json.dumps({"arguments":arguments,"result":result})) + assert completed.returncode==0,result + return result +common=["--goal-id",os.environ["LOOPX_GOAL_ID"],"--agent-id",os.environ["LOOPX_AGENT_ID"], + "--turn-instance-id",os.environ["LOOPX_TURN"]] +guard=cli("quota","should-run","--runtime-profile","generic_cli",*common) +assert guard["should_run"] is True,guard +selected=guard["selected_todo"] +text=selected["text"] +phase=re.search(r"Phase (S[12])",text).group(1) +inputs=json.loads(Path("retained-inputs.json").read_text()) +history=Path("native-history.jsonl") +if phase=="S2": + assert "resume" in args and "retained-phase-session" in args,args + old=json.loads(history.read_text().splitlines()[0]) + assert old["phase"]=="S1" and old["old_deadline"]float(os.environ["FIXTURE_HOST_NOW"]) + assert "sum of squares" in text,text + value=sum(number*number for number in inputs) +else: + assert "sum of retained inputs" in text,text + value=sum(inputs) +assert value==(5 if phase=="S1" else 13) +artifact=Path("probe-"+phase+".json") +artifact.write_text(json.dumps({"phase":phase,"inputs":inputs,"value":value, + "consumed_todo_id":selected["todo_id"]})+"\n") +with history.open("a") as stream: + stream.write(json.dumps({"phase":phase,"old_deadline":int(os.environ["FIXTURE_S1_DEADLINE"]),"value":value})+"\n") +with Path("model-counts.jsonl").open("a") as stream: + stream.write(json.dumps({"phase":phase,"D":1,"P":1})+"\n") +print(json.dumps({"type":"thread.started","thread_id":"retained-phase-session"}),flush=True) +evidence="fixture:phase-probe:"+sha256(artifact.read_bytes()).hexdigest() +binding=[*common,"--todo-id",selected["todo_id"]] +completion=["todo","complete",*binding,"--claimed-by",os.environ["LOOPX_AGENT_ID"],"--evidence",evidence] +cli(*completion) +vision={"schema_version":"goal_vision_replan_contract_v0","state":"no_followup", + "vision_patch":{"vision_summary":"Execute the explicitly authorized finite phase probe.", + "acceptance_summary":"The phase probe ran on retained inputs and matched its exact expected value."}, + "path_delta":{"schema_version":"goal_path_delta_v0","outcome":"no_change", + "prior_assumption":"The currently authorized phase probe remained unexecuted.", + "observed_reality":"The probe ran on retained inputs and its exact result passed validation.", + "stopped":["No further work belongs to this finite phase probe."],"evidence_refs":[evidence]}} +vision_path=Path("phase-"+phase+".vision.json").resolve() +vision_path.write_text(json.dumps(vision)) +refreshed=cli("refresh-state",*binding,"--classification","phase_probe_completed", + "--delivery-batch-scale","implementation","--delivery-outcome","outcome_progress", + "--agent-vision-json",str(vision_path),"--no-global-sync","--suppress-external-sinks") +spent=cli("quota","spend-slot",*binding,"--slots","1","--source","heartbeat","--execute") +terminal=cli(*completion,"--no-follow-up") +assert spent["appended"] is True and terminal["completion_continuation"]=="no_followup" +Path("phase-"+phase+"-settlement.json").write_text(json.dumps({"binding":binding, + "completion":completion,"identity":spent["settlement_identity"],"evidence":evidence, + "refreshed":refreshed,"spent":spent,"terminal":terminal})) +''' + + +def _spends_for_todo(index, todo_id): + prefix = f"{GOAL}:{AGENT}:{todo_id}:" + return [row for row in _spends(index) + if row.get("quota_spend_commit", {}).get("effect_id", "").startswith(prefix)] + + +def _argv(project, runtime, registry, native, phase): + validation = ("import json; from pathlib import Path; " + f"r=json.loads(Path('probe-{phase}.json').read_text()); " + f"assert r['phase']=={phase!r} and r['value']=={5 if phase == 'S1' else 13}") + return ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", + "turn", "run-once", "--host", "codex-cli", "--codex-bin", str(native), + "--goal-id", GOAL, "--agent-id", AGENT, "--project", str(project), "--scan-root", str(project), + "--iteration-context", "resume", "--validation-command-json", json.dumps([sys.executable, "-c", validation]), + "--timeout-seconds", "120", "--no-global-sync", "--execute"] + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("mode", ["governed", "heartbeat"]) +def test_current_phase_probe_executes_once_after_unique_extension(controllers, tmp_path, monkeypatch, provider, mode): + isolate_sqlite_runtime(tmp_path / "canonical", monkeypatch) + project, runtime, registry = fixture._write_live_fixture(tmp_path / "goal", provider=provider) + monkeypatch.setenv("PYTHONPATH", str(ROOT)) + workspace = project / "native-workspace" + workspace.mkdir() + (workspace / "retained-inputs.json").write_text(json.dumps([2, 3])) + native = tmp_path / "controlled-codex" + native.write_text(f"#!{sys.executable}\n" + (_PHASE_HOST if mode == "governed" else _HEARTBEAT_PHASE_HOST)) + native.chmod(0o700) + environment = worker_env(workspace) + skills = project / "shared-skills" + skills.mkdir() + launcher = project / "phase-loopx" + launcher.write_text("#!/bin/sh\nexec " + shlex.join([sys.executable, "-m", "loopx.cli"]) + ' "$@"\n') + launcher.chmod(0o700) + environment.update(LOOPX_EXECUTION_MODE="heartbeat", LOOPX_ITERATION_CONTEXT="resume", + LOOPX_RUNTIME_ROOT=str(runtime), LOOPX_REGISTRY=str(registry), LOOPX_GOAL_ID=GOAL, + LOOPX_AGENT_ID=AGENT, LOOPX_SHARED_SKILLS=str(skills), LOOPX_CLI=str(launcher), + LOOPX_TURN_ENVELOPE="0", LOOPX_CODEX_SANDBOX="read-only", CODEX_BIN=str(native), + LOOPX_CODEX_TURN_TIMEOUT_SEC="120") + + def run_phase(phase): + if mode == "governed": + result = _in_process(_argv(workspace, runtime, registry, native, phase)) + assert result["status"] == "committed" + return result + environment.update({key: os.environ[key] for key in ( + "FIXTURE_S1_DEADLINE", "FIXTURE_HOST_NOW", "FIXTURE_PHASE_FILE") if key in os.environ}) + environment["LOOPX_PHASE_DEADLINE_EPOCH"] = ( + os.environ["FIXTURE_S1_DEADLINE"] if phase == "S1" + else Path(os.environ["FIXTURE_PHASE_FILE"]).read_text().strip()) + receipt = benchmark_worker.run_once(environment) + error_path = workspace / "phase-host-error.json" + assert receipt["ok"] is True, error_path.read_text() if error_path.exists() else receipt + assert receipt["host_invoked"] is True and receipt["session"]["session_id"] == "retained-phase-session" + return receipt + _cli(registry, runtime, "todo", "update", "--goal-id", GOAL, "--agent-id", AGENT, + "--todo-id", TODO, "--text", "Phase S1: compute the sum of retained inputs.") + source_phase = tmp_path / "source-phase" + source_phase.write_text(f"{int(time.time()) + 1800}\n") + monkeypatch.setenv("FIXTURE_S1_DEADLINE", source_phase.read_text().strip()) + run_phase("S1") + old_artifact = (workspace / "probe-S1.json").read_bytes() + old_history = (workspace / "native-history.jsonl").read_bytes() + index = runtime / "goals" / GOAL / "runs/index.jsonl" + original_spends = _spends(index) + assert len(original_spends) == 1 + + source, destination, source_url, destination_url, files, grades = controllers + candidate_policy = (tmp_path / "tasks/fixture.json").read_bytes() + assert json.loads(candidate_policy)["judge"]["eval_timeout"] == 15 + token = register(source, source_url, max_agent_submissions=3, submission_cooldown=3600) + assert submit(source_url, token, 7).status_code == 200 + original_phase = source_phase.read_bytes() + unit = tmp_path / "retained-point" + point = transport.capture(files[0], token=token, capture_id="e" * 32, + destination=unit, phase_deadline_file=source_phase) + # Advance only the receiving Host clock. HTTP, source bytes and candidate + # grading remain real; the old deadline is past under this clock. + host_clock = time.time() + 3600 + monkeypatch.setattr(transport, "time", SimpleNamespace( + time=lambda: host_clock, monotonic=time.monotonic, sleep=time.sleep)) + monkeypatch.setattr(benchmark_worker, "time", SimpleNamespace( + time=lambda: host_clock, monotonic=time.monotonic, sleep=time.sleep, + time_ns=lambda: int(host_clock * 1_000_000_000))) + monkeypatch.setenv("FIXTURE_HOST_NOW", str(host_clock)) + point_bytes = (unit / "checkpoint.json").read_bytes() + # A retained point does not carry authority to restart a stopped Goal. + stopped = set_goal_activation_state(registry_path=registry, goal_id=GOAL, + state="stopped", reason="The first finite research phase stopped.", + actor_kind="owner", execute=True) + assert stopped["readback"]["verified"] is True + stopped_guard = _cli(registry, runtime, "quota", "should-run", "--runtime-profile", "generic_cli", + "--goal-id", GOAL, "--agent-id", AGENT) + assert stopped_guard["should_run"] is False and stopped_guard["pause_cause"] == "goal_stopped" + if mode == "governed": + stopped_turn = _in_process(_argv(workspace, runtime, registry, native, "S2"), code=1) + assert stopped_turn["effects"]["host_invoked"] is False + else: + expired = benchmark_worker.run_once(environment) + assert expired["budget_exhausted"] is True and expired["host_invoked"] is False + assert len((workspace / "model-counts.jsonl").read_text().splitlines()) == 1 + assert _spends(index) == original_spends + + phase_request = {"event_id": "user-continue-phase-s2", "phase": "S2", "extension_seconds": 600, + "probe": "compute the sum of squares of retained inputs"} + phase = tmp_path / "current-phase" + restored = transport.restore(files[1], root=unit, expected_sha256=point["sha256"], + run_id=phase_request["event_id"], phase_deadline_file=phase, + phase_extension_seconds=phase_request["extension_seconds"]) + deadline = int(phase.read_text()) + assert deadline > host_clock + monkeypatch.setenv("FIXTURE_PHASE_FILE", str(phase)) + binding = phase.with_name(phase.name + ".restore.json").read_bytes() + add = ("todo", "add", "--goal-id", GOAL, "--role", "agent", "--claimed-by", AGENT, + "--task-class", "advancement_task", "--action-kind", "validate", + "--text", "Phase S2: compute the sum of squares of retained inputs.", + "--operation-id", phase_request["event_id"]) + resumed = set_goal_activation_state(registry_path=registry, goal_id=GOAL, + state="active", reason=phase_request["event_id"] + ": " + phase_request["probe"], + actor_kind="owner", execute=True) + assert resumed["readback"]["verified"] is True + current_todo = _cli(registry, runtime, *add)["todo_id"] + assert _cli(registry, runtime, *add)["todo_id"] == current_todo + assert transport.restore(files[1], root=unit, expected_sha256=point["sha256"], + run_id=phase_request["event_id"], phase_deadline_file=phase, recover=True, + phase_extension_seconds=phase_request["extension_seconds"]) == restored + assert phase.with_name(phase.name + ".restore.json").read_bytes() == binding + info = destination.state.judge.tokens[restored["token"]] + assert info["max_agent_submissions"] == 3 and info["submission_cooldown"] == 3600 + assert submit(destination_url, restored["token"], 8).status_code == 429 + assert len(grades) == 1 + + second_argv = _argv(workspace, runtime, registry, native, "S2") + second = run_phase("S2") + artifact = json.loads((workspace / "probe-S2.json").read_text()) + assert artifact == {"phase": "S2", "inputs": [2, 3], "value": 13, "consumed_todo_id": current_todo} + assert (workspace / "probe-S1.json").read_bytes() == old_artifact + assert (workspace / "native-history.jsonl").read_bytes().startswith(old_history) + current_spends = _spends_for_todo(index, current_todo) + assert len(current_spends) == 1 + todo = _cli(registry, runtime, "todo", "list", "--goal-id", GOAL, "--todo-id", current_todo) + assert todo["todo"]["status"] == "done" + assert todo["authority_read"]["source_authority"] == f"{provider}_v0" + for _ in range(2): + assert _cli(registry, runtime, *add)["todo_id"] == current_todo + if mode == "governed": + replay = _in_process([*second_argv[:-1], "--resume-turn-key", second["resume_turn_key"], "--execute"]) + assert replay["effects"]["host_invoked"] is False + else: + original = json.loads((workspace / "phase-S2-settlement.json").read_text()) + replay = _cli(registry, runtime, *original["completion"], "--no-follow-up") + assert replay["idempotent_replay"] is True + replay = _cli(registry, runtime, "quota", "spend-slot", *original["binding"], + "--slots", "1", "--source", "heartbeat", "--execute") + assert replay["appended"] is False and replay["idempotent_replay"] is True + transport.restore(files[1], root=unit, expected_sha256=point["sha256"], + run_id=phase_request["event_id"], phase_deadline_file=phase, recover=True, + phase_extension_seconds=phase_request["extension_seconds"]) + assert int(phase.read_text()) == deadline + assert source_phase.read_bytes() == original_phase and (unit / "checkpoint.json").read_bytes() == point_bytes + assert [json.loads(line)["phase"] for line in (workspace / "model-counts.jsonl").read_text().splitlines()] == ["S1", "S2"] + assert _spends_for_todo(index, TODO) == original_spends + assert _spends_for_todo(index, current_todo) == current_spends + assert (tmp_path / "tasks/fixture.json").read_bytes() == candidate_policy + assert len(destination.state.judge.tokens) == 1 + if mode == "heartbeat": + after = _cli(registry, runtime, "quota", "should-run", "--runtime-profile", "generic_cli", + "--goal-id", GOAL, "--agent-id", AGENT) + assert after["should_run"] is False and after.get("selected_todo") is None, after + with pytest.raises(ValueError, match="original activation"): + transport.restore(files[1], root=unit, expected_sha256=point["sha256"], + run_id=phase_request["event_id"], phase_deadline_file=phase, recover=True, + phase_extension_seconds=601) diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index 7d6bd31ed5..82ac823d7a 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -130,6 +130,7 @@ const CANONICAL_CONSUMERS = [ "control_plane/goals/acceptance_contract.ts", "control_plane/goals/acceptance_oracle.ts", "control_plane/goals/direction_authority.ts", + "control_plane/goals/first_party_host_runtime.ts", "control_plane/goals/goal_amendment_authority.ts", "control_plane/goals/goal_amendment_proposal.ts", "control_plane/goals/operator_actions.ts", diff --git a/tests/control_plane_ts/first_party_host_runtime.test.ts b/tests/control_plane_ts/first_party_host_runtime.test.ts index 4e98d42e88..edfedc95cb 100644 --- a/tests/control_plane_ts/first_party_host_runtime.test.ts +++ b/tests/control_plane_ts/first_party_host_runtime.test.ts @@ -9,7 +9,7 @@ const GOAL_A = { goal_id: "release", goal_instance_id: INSTANCE_A }; const GOAL_B = { goal_id: "release", goal_instance_id: INSTANCE_B }; function facts( - operation: "select_state" | "require_current" | "accept_result", + operation: "select_state" | "require_current" | "launch" | "accept_result", overrides: Record = {}, ) { return { @@ -41,6 +41,73 @@ test("non-source profiles preserve legacy behavior without decoding strict facts ); }); +test("legacy Host launch requires the original live registry witness without inventing a GoalRef", () => { + const witness = {path: "/fixture/registry.json", sha256: "a".repeat(64)}; + const launch = { + profile_id: null, operation: "launch", goal_id: "release", registry_path: witness.path, + planned_registry_witness: witness, registry_witness: witness, + authority: {kind: "present", goal_id: "release", registration_status: "active", activation_state: "active"}, + }; + assert.deepEqual(decideFirstPartyHostRuntime(launch), {kind: "legacy"}); + for (const status of ["active-read-only", " ACTIVE-READ-ONLY ", "ACTIVE", " ACTIVE-research "]) { + const authority = {...launch.authority, registration_status: status}; + assert.deepEqual(decideFirstPartyHostRuntime({...launch, authority}), {kind: "legacy"}); + assert.deepEqual(decideFirstPartyHostRuntime({ + ...launch, authority: {...authority, activation_state: "stopped"}, + }), {kind: "reject", code: "goal_stopped"}); + assert.deepEqual(decideFirstPartyHostRuntime({ + ...launch, authority, registry_witness: {...witness, sha256: "b".repeat(64)}, + }), {kind: "reject", code: "stale_goal_authority"}); + } + for (const status of ["activeish", "inactive", "retired", null, 1]) { + assert.deepEqual(decideFirstPartyHostRuntime({ + ...launch, authority: {...launch.authority, registration_status: status}, + }), {kind: "reject", code: "goal_not_registered"}); + } + for (const [authority, code] of [ + [{kind: "absent"}, "goal_not_registered"], + [{kind: "unavailable"}, "goal_authority_unavailable"], + [{...launch.authority, activation_state: "stopped"}, "goal_stopped"], + [{...launch.authority, activation_state: "invalid"}, "goal_authority_unavailable"], + [{...launch.authority, registration_status: "retired"}, "goal_not_registered"], + [{...launch.authority, goal_id: "other"}, "goal_authority_unavailable"], + ] as const) { + assert.deepEqual(decideFirstPartyHostRuntime({...launch, authority}), {kind: "reject", code}); + } + for (const changed of [{...witness, sha256: "b".repeat(64)}, {...witness, path: "/other/registry.json"}]) { + assert.deepEqual(decideFirstPartyHostRuntime({...launch, registry_witness: changed}), + {kind: "reject", code: "stale_goal_authority"}); + } + for (const missing of [null, {}, {...witness, sha256: "not-a-digest"}]) { + assert.deepEqual(decideFirstPartyHostRuntime({...launch, planned_registry_witness: missing}), + {kind: "reject", code: "goal_authority_unavailable"}); + } +}); + +test("legacy native launch consumes the current quota owner without weakening source identity", () => { + const witness = {path: "/fixture/registry.json", sha256: "a".repeat(64)}; + const legacy = {profile_id: null, operation: "launch", goal_id: "release", + registry_path: witness.path, planned_registry_witness: witness, registry_witness: witness, + authority: {kind: "present", goal_id: "release", registration_status: "active", activation_state: "active"}}; + const exact = {...facts("require_current"), operation: "launch"}; + const admitted = {planned_effective_action: "normal_run", current: { + should_run: true, status_health_ok: true, effective_action: "normal_run"}}; + for (const launch_decision of [null, {}, {...admitted, current: {...admitted.current, should_run: false}}, + {...admitted, current: {...admitted.current, effective_action: "control_plane_health_repair"}}, + {...admitted, current: {...admitted.current, should_run: "true"}}]) { + assert.deepEqual(decideFirstPartyHostRuntime({...legacy, launch_decision}), + {kind: "reject", code: "goal_launch_not_allowed"}); + } + assert.notEqual(decideFirstPartyHostRuntime({...legacy, launch_decision: admitted}).kind, "reject"); + assert.notEqual(decideFirstPartyHostRuntime({...legacy, launch_decision: { + planned_effective_action: "control_plane_health_repair", + current: {should_run: true, status_health_ok: false, effective_action: "control_plane_health_repair"}, + }}).kind, "reject"); + assert.deepEqual(decideFirstPartyHostRuntime({...exact, + planned_goal_ref: {goal_id: "release"}, launch_decision: admitted}), + {kind: "reject", code: "goal_instance_id_missing"}); +}); + test("current exact GoalRef admits each host runtime checkpoint", () => { assert.deepEqual(decideFirstPartyHostRuntime(facts("select_state")), { kind: "start_new", @@ -58,6 +125,10 @@ test("current exact GoalRef admits each host runtime checkpoint", () => { kind: "resume", goal_ref: GOAL_A, }); + assert.deepEqual(decideFirstPartyHostRuntime(facts("launch")), { + kind: "resume", + goal_ref: GOAL_A, + }); assert.deepEqual(decideFirstPartyHostRuntime(facts("accept_result")), { kind: "accept_result", goal_ref: GOAL_A, @@ -68,6 +139,7 @@ test("Goal recreation rejects a stale plan at every checkpoint", () => { for (const operation of [ "select_state", "require_current", + "launch", "accept_result", ] as const) { assert.deepEqual( @@ -86,6 +158,7 @@ test("a stopped exact Goal rejects every host runtime checkpoint", () => { for (const operation of [ "select_state", "require_current", + "launch", "accept_result", ] as const) { assert.deepEqual( diff --git a/tests/test_codex_operation_host.py b/tests/test_codex_operation_host.py index 9397ab7238..f57808c21e 100644 --- a/tests/test_codex_operation_host.py +++ b/tests/test_codex_operation_host.py @@ -21,6 +21,7 @@ run_codex_operation_host, ) from loopx.control_plane.turn_driver.host_failure import BuiltInHostError +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected from loopx.control_plane.turn_driver.executor import validate_loopx_turn_host_result from test_loopx_turn_codex_cli import _request from test_loopx_turn_driver import _write_live_fixture @@ -88,6 +89,18 @@ def emit(value): """ +def _registered_host_goal(tmp_path: Path) -> Path: + """Direct transport tests still require a registered live Goal at launch.""" + registry = tmp_path / "registry.json" + registry.write_text(json.dumps({ + "schema_version": 1, "common_runtime_root": str(tmp_path / "runtime"), + "goals": [{"id": "fixture-goal", "status": "active", "repo": str(tmp_path), + "quota": {"compute": 1, "window_hours": 24}, + "coordination": {"registered_agents": ["codex-fixture"]}}], + })) + return registry + + def _claimed_native_fixture(tmp_path: Path): from examples import operation_action_fixtures as fixtures from loopx.control_plane.turn_driver.codex_operation_host import operation_tool_handler @@ -480,7 +493,7 @@ def test_owned_app_server_process_authenticates_native_metadata_and_resumes_same executable.chmod(0o700) options = { "runtime_root": tmp_path / "runtime", - "registry_path": tmp_path / "registry.json", + "registry_path": _registered_host_goal(tmp_path), "project": tmp_path, "codex_bin": str(executable), "model": "test-model", @@ -606,6 +619,34 @@ def test_operation_host_refuses_unpinned_or_widened_profile_before_launch( assert not (tmp_path / "runtime").exists() +@pytest.mark.parametrize("registered", [False, None]) +def test_operation_host_rejects_absent_goal_authority_before_native_start( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, registered, +) -> None: + from loopx import chat_agent + + registry = tmp_path / "registry.json" + if registered is False: + registry.write_text(json.dumps({"schema_version": 1, "goals": []})) + executable = tmp_path / "fake-codex-operation" + executable.write_text(FAKE_SERVER) + executable.chmod(0o700) + start = chat_agent.subprocess.Popen + + def forbidden_start(command, *args, **kwargs): + if command[0] == str(executable): + pytest.fail("missing Goal authority cannot create a native process") + return start(command, *args, **kwargs) + + monkeypatch.setattr("loopx.chat_agent.subprocess.Popen", forbidden_start) + with pytest.raises(FirstPartyHostRuntimeRejected): + run_codex_operation_host( + _request(), registry_path=registry, runtime_root=tmp_path / "runtime", + project=tmp_path, codex_bin=str(executable), model="test-model", + reasoning_effort="xhigh", + ) + + @pytest.mark.skipif(os.name != "posix", reason="POSIX process-group qualification") @pytest.mark.parametrize("hang", [False, True]) def test_owned_operation_host_reaps_descendants_on_success_and_timeout( @@ -620,7 +661,7 @@ def test_owned_operation_host_reaps_descendants_on_success_and_timeout( monkeypatch.setenv("FAKE_OPERATION_HANG", "1") options = dict( runtime_root=tmp_path / "runtime", - registry_path=tmp_path / "registry.json", + registry_path=_registered_host_goal(tmp_path), project=tmp_path, codex_bin=str(executable), model="test-model", @@ -673,7 +714,7 @@ def record(tool, arguments, native): ) options = dict( runtime_root=tmp_path / "runtime", - registry_path=tmp_path / "registry.json", + registry_path=_registered_host_goal(tmp_path), project=tmp_path, model=model, reasoning_effort=effort, diff --git a/tests/test_codex_operation_launch_admission.py b/tests/test_codex_operation_launch_admission.py new file mode 100644 index 0000000000..a11d8d1f29 --- /dev/null +++ b/tests/test_codex_operation_launch_admission.py @@ -0,0 +1,88 @@ +"""Actual managed CLI/native boundaries, with a deterministic local app-server.""" +import contextlib +import io +import json +import sys + +import pytest + +from loopx.cli import main +from loopx.control_plane.goals.activation_service import set_goal_activation_state +from loopx.control_plane.turn_driver import codex_operation_host +from test_codex_operation_host import FAKE_SERVER +from test_loopx_turn_driver import _write_live_fixture +from test_loopx_turn_codex_cli import _fake_codex + + +@pytest.mark.parametrize("boundary", ["process", "turn"]) +def test_owner_stop_prevents_operation_native_launch(tmp_path, monkeypatch, boundary): + project, runtime, registry = _write_live_fixture(tmp_path) + process_marker, turn_marker = tmp_path / "process-started", tmp_path / "turn-started" + native = tmp_path / "controlled-codex" + server = FAKE_SERVER.replace("import json, sys", + f"import json, sys\nfrom pathlib import Path\nPath({str(process_marker)!r}).touch()", 1) + server = server.replace('elif method == "turn/start":', + f'elif method == "turn/start":\n Path({str(turn_marker)!r}).touch()', 1) + native.write_text(server) + native.chmod(0o700) + original_start = codex_operation_host.CodexChatAgentSession.start + stops = [] + + def stop(): + result = set_goal_activation_state(registry_path=registry, goal_id="loopx-turn-fixture", + state="stopped", reason="Synthetic owner cancellation before native launch.", + actor_kind="owner", execute=True) + assert result["readback"]["verified"] is True + stops.append(True) + + def intercepted_start(**kwargs): + if boundary == "process": + stop() + session = original_start(**kwargs) + if boundary == "turn": + # The registry lock must already be released after process start: + # the real writer can complete before the next model boundary. + stop() + return session + + monkeypatch.setattr(codex_operation_host.CodexChatAgentSession, "start", intercepted_start) + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = main([ + "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", + "turn", "run-once", "--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture", + "--host", "codex-cli", "--project", str(project), "--scan-root", str(project), + "--no-global-sync", "--codex-operation-tools", "--codex-bin", str(native), + "--codex-model", "test-model", "--codex-reasoning-effort", "xhigh", + "--codex-sandbox", "read-only", "--iteration-context", "fresh", + "--timeout-seconds", "15", "--execute", + ]) + packet = json.loads(output.getvalue()) + assert code != 0, packet + assert stops == [True] + assert process_marker.exists() == (boundary == "turn") + assert not turn_marker.exists() + assert not packet["effects"]["state_written"] and not packet["effects"]["quota_spent"] + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +def test_legacy_read_only_registration_can_preview_and_launch(tmp_path, monkeypatch): + project, runtime, registry = _write_live_fixture(tmp_path) + value = json.loads(registry.read_text()) + value["goals"][0]["status"] = "active-read-only" + registry.write_text(json.dumps(value)) + native, marker = _fake_codex(tmp_path) + monkeypatch.setenv("FAKE_CODEX_LOG", str(marker)) + args = ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", + "turn", "run-once", "--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture", + "--host", "codex-cli", "--project", str(project), "--scan-root", str(project), + "--codex-bin", str(native), "--codex-sandbox", "read-only", "--no-global-sync", + "--iteration-context", "fresh", "--validation-command-json", json.dumps([sys.executable, "-c", "pass"])] + for suffix in ([], ["--execute"]): + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = main([*args, *suffix]) + packet = json.loads(output.getvalue()) + assert code == 0 and packet["ok"], packet + assert packet["status"] == "committed" and marker.exists() diff --git a/tests/test_host_launch_authority.py b/tests/test_host_launch_authority.py new file mode 100644 index 0000000000..0571e78dd0 --- /dev/null +++ b/tests/test_host_launch_authority.py @@ -0,0 +1,375 @@ +"""Actual CLI/native launch boundaries; disposable Goals, no live model calls.""" + +from __future__ import annotations + +import contextlib +import io +import json +import sys +import threading +from pathlib import Path + +import pytest + +from loopx.cli import main as cli_main +from loopx.cli_commands import lark_inbox, turn_decision +from loopx.configure_goal import configure_goal +from loopx.control_plane.goals import activation_service +from loopx.control_plane.projects.registry_codec import ( + project_registry_transaction, + source_session_registry_transaction, +) +from loopx.control_plane.turn_driver import codex_cli, codex_operation_host +from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime +from tests.control_plane.test_effect_turn_live_quota_decision import ( + _material_review_urgency, + _turn_start_dispatch, +) +from test_codex_operation_host import FAKE_SERVER +from test_loopx_turn_codex_cli import _fake_codex, _session_files, _source_admission +from test_loopx_turn_driver import _write_live_fixture + + +GOAL = "loopx-turn-fixture" +AGENT = "codex-fixture" + + +def _cli(registry: Path, runtime: Path, *arguments: str): + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = cli_main([ + "--registry", str(registry), "--runtime-root", str(runtime), + "--format", "json", *arguments, + ]) + return code, json.loads(output.getvalue()) + + +def _mirror(source: Path, runtime: Path) -> Path: + registry = runtime / "registry.global.json" + payload = json.loads(source.read_text()) + payload["registry_role"] = "global-local" + payload["goals"][0]["source_registry"] = str(source) + registry.write_text(json.dumps(payload)) + return registry + + +def _host(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, operation_tools: bool): + native_start, turn_start = tmp_path / "native-start", tmp_path / "turn-start" + if operation_tools: + executable = tmp_path / "fake-codex-operation" + executable.write_text(FAKE_SERVER.replace( + "import json, sys", "import json, sys, pathlib\n" + f"pathlib.Path({str(native_start)!r}).write_text('started')", 1, + ).replace( + 'elif method == "turn/start":', 'elif method == "turn/start":\n' + f" pathlib.Path({str(turn_start)!r}).write_text('started')", 1, + )) + executable.chmod(0o700) + else: + executable, turn_start = _fake_codex(tmp_path) + executable.write_text(executable.read_text().replace( + "args = sys.argv[1:]", + f"pathlib.Path({str(native_start)!r}).write_text('started')\nargs = sys.argv[1:]", 1, + )) + monkeypatch.setenv("FAKE_CODEX_LOG", str(turn_start)) + arguments = ["--codex-bin", str(executable)] + if operation_tools: + arguments.extend(["--codex-operation-tools", "--codex-model", "test-model", + "--codex-reasoning-effort", "xhigh"]) + return arguments, native_start, turn_start + + +def _run(registry: Path, runtime: Path, project: Path, host_arguments: list[str]): + return _cli(registry, runtime, + "turn", "run-once", "--goal-id", GOAL, "--agent-id", AGENT, + "--host", "codex-cli", "--project", str(project), "--scan-root", str(project), + "--codex-sandbox", "read-only", "--no-global-sync", "--iteration-context", "fresh", + "--turn-instance-id", "launch-authority-fixture", + "--validation-command-json", json.dumps([sys.executable, "-c", "pass"]), + *host_arguments, "--execute", + ) + + +def _assert_no_settlement(runtime: Path, payload: dict): + assert not payload["effects"]["state_written"], payload + assert not payload["effects"]["quota_spent"], payload + index = runtime / f"goals/{GOAL}/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + + +@pytest.mark.parametrize("provider,route", [("file", "global"), ("sqlite", "project"), ("sqlite", "global")]) +@pytest.mark.parametrize("boundary", ["process", "turn"]) +def test_operation_cli_observes_verified_stop_at_each_native_boundary( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, route: str, boundary: str, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, source = _write_live_fixture(tmp_path, provider=provider) + registry = _mirror(source, runtime) if route == "global" else source + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, True) + stops = [] + + def stop(): + result = activation_service.set_goal_activation_state( + registry_path=registry, runtime_root_override=str(runtime), goal_id=GOAL, + state="stopped", reason="Synthetic owner Stop before native dispatch.", + actor_kind="owner", execute=True, + ) + assert result["readback"]["verified"] is True, result + stops.append(result) + + if boundary == "process": + start = codex_operation_host.CodexChatAgentSession.start + + def stopped_start(**arguments): + stop() + return start(**arguments) + + monkeypatch.setattr(codex_operation_host.CodexChatAgentSession, "start", stopped_start) + else: + prompt = codex_operation_host._prompt + + def stopped_prompt(request): + # App-server/thread initialization already completed, but the + # request must never cross the native model Turn boundary. + assert native_start.exists() + stop() + return prompt(request) + + monkeypatch.setattr(codex_operation_host, "_prompt", stopped_prompt) + code, payload = _run(registry, runtime, project, host_arguments) + assert stops and code == 1 and "goal_stopped" in json.dumps(payload), payload + assert native_start.exists() is (boundary == "turn") + assert not turn_start.exists(), payload + if boundary == "process": + assert not _session_files(runtime) + _assert_no_settlement(runtime, payload) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("operation_tools", [False, True]) +@pytest.mark.parametrize("drift", ["source_quota", "source_health", "global_health", "local_health"]) +def test_actual_launch_consumes_current_source_quota_and_health( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, + operation_tools: bool, drift: str, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, source = _write_live_fixture(tmp_path, provider=provider) + mirror = _mirror(source, runtime) if drift != "local_health" else None + registry = source if drift in {"source_health", "local_health"} else mirror + assert registry is not None + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, operation_tools) + before_registry = registry.read_bytes() + + if drift == "source_quota": + configure_goal(registry_path=source, goal_id=GOAL, quota_compute=0, execute=True) + assert json.loads(source.read_text())["goals"][0]["quota"]["compute"] == 0 + assert registry.read_bytes() == before_registry + code, current = _cli(source, runtime, "quota", "should-run", "--goal-id", GOAL, + "--agent-id", AGENT, "--runtime-profile", "generic_cli") + assert code == 0 and current["should_run"] is False + else: + state = project / f".codex/goals/{GOAL}/ACTIVE_GOAL_STATE.md" + + def remove_state(): + state.rename(state.with_suffix(".removed")) + assert registry.read_bytes() == before_registry + + if operation_tools: + start = codex_operation_host.CodexChatAgentSession.start + + def missing_state_start(**arguments): + remove_state() + return start(**arguments) + + monkeypatch.setattr(codex_operation_host.CodexChatAgentSession, "start", missing_state_start) + else: + prompt = codex_cli._prompt + + def missing_state_prompt(request): + assert request["turn_envelope"]["action"]["selected_todo"]["todo_id"] == "todo_fixture0001" + remove_state() + return prompt(request) + + monkeypatch.setattr(codex_cli, "_prompt", missing_state_prompt) + + code, payload = _run(registry, runtime, project, host_arguments) + assert code == 1 and "goal_launch_not_allowed" in json.dumps(payload), payload + assert not native_start.exists() and not turn_start.exists(), payload + assert not _session_files(runtime) + _assert_no_settlement(runtime, payload) + if drift == "local_health": + assert not (runtime / "registry.global.json").exists() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_first_normal_turn_with_declared_state_needs_no_global_registry( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider) + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, False) + assert not (runtime / "registry.global.json").exists() + assert not _session_files(runtime) + code, payload = _run(registry, runtime, project, host_arguments) + assert code == 0 and payload["status"] == "committed", payload + assert native_start.exists() and turn_start.exists(), payload + assert payload["effects"]["quota_spent"] is True, payload + assert not (runtime / "registry.global.json").exists() + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("operation_tools", [False, True]) +@pytest.mark.parametrize("late_material", [False, True]) +def test_actual_launch_preserves_same_turn_inbox_read_scope( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation_tools: bool, + late_material: bool, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider) + state = project / f".codex/goals/{GOAL}/ACTIVE_GOAL_STATE.md" + original_state = state.read_bytes() + with project_registry_transaction(registry, operation="synthetic_fresh_inbox") as tx: + current = tx.payload_copy() + current["goals"][0].setdefault("control_plane", {})["lark_event_inbox"] = { + "enabled": True, "config_path": ".loopx/config/inbox.json", + } + tx.commit(current) + monkeypatch.setattr( + "loopx.control_plane.quota.goal_boundary.operator_inbox_binding", + lambda **_kwargs: {"status": "verified", "attention_required": False}, + ) + pending_count = 1 + + def urgency(**_kwargs): + return _material_review_urgency(pending_count=pending_count) + + monkeypatch.setattr(turn_decision, "build_lark_operator_inbox_urgency_projector", + lambda **_kwargs: urgency) + dispatches = [] + + def sync(**_kwargs): + dispatch = _turn_start_dispatch(commands=(f"loopx inbox drain --goal-id {GOAL}",)) + dispatches.append(dispatch) + return dispatch + + monkeypatch.setattr(lark_inbox, "dispatch_goal_lark_turn_start_hooks", sync) + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, operation_tools) + if late_material: + owner = codex_operation_host if operation_tools else codex_cli + prompt = owner._prompt + + def changed_inbox_prompt(request): + nonlocal pending_count + # A later message was not in this Turn's initial required read. + # Operation Host is already open here, before its model Turn. + pending_count = 2 + return prompt(request) + + monkeypatch.setattr(owner, "_prompt", changed_inbox_prompt) + code, payload = _run(registry, runtime, project, host_arguments) + assert len(dispatches) == 1 + if late_material: + assert code == 1 and "goal_launch_not_allowed" in json.dumps(payload), payload + assert native_start.exists() is operation_tools + assert not turn_start.exists(), payload + # Keep the already performed inbox sync visible even when launch is + # refused. It cannot stand in for work writeback or quota settlement. + assert payload["effects"]["state_written"] is True, payload + assert not payload["effects"]["quota_spent"], payload + # Failed execution may keep an effect unknown; inspect its actual + # business state and spend history instead of upgrading that readback. + assert state.read_bytes() == original_state + index = runtime / f"goals/{GOAL}/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() + return + assert code == 0, payload + # The shared operation server requests confirmation and returns a wait; + # retaining fresh reads grants no additional operation or settlement. + assert payload["status"] == ("stopped" if operation_tools else "committed"), payload + assert native_start.exists() and turn_start.exists(), payload + assert payload["effects"]["host_invoked"] is True, payload + # New material belongs to this Turn's required reads. Launch checks must + # retain that context without synchronizing the external source again. + assert payload["effects"]["quota_spent"] is (not operation_tools), payload + index = runtime / f"goals/{GOAL}/runs/index.jsonl" + rows = [json.loads(line) for line in index.read_text().splitlines()] if index.exists() else [] + assert sum(row.get("classification") == "quota_slot_spent" for row in rows) == int(not operation_tools) + + +@pytest.mark.parametrize("operation_tools", [False, True]) +def test_source_session_cli_keeps_lifecycle_only_execution_denial( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, operation_tools: bool, +): + admission = _source_admission(tmp_path) + with source_session_registry_transaction( + admission.registry_path, operation="fixture_cli_source_goal", + ) as transaction: + registry = transaction.payload_copy() + registry["goals"][0]["id"] = GOAL + transaction.commit(registry) + before = admission.registry_path.read_bytes() + runtime = tmp_path / "runtime" + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, operation_tools) + code, payload = _run(admission.registry_path, runtime, tmp_path, host_arguments) + assert code == 1 and "lifecycle-only profile source_session_v1" in json.dumps(payload), payload + assert not native_start.exists() and not turn_start.exists() + assert not _session_files(runtime) + assert admission.registry_path.read_bytes() == before + + +@pytest.mark.parametrize("provider,operation_tools", [("file", True), ("sqlite", False), ("sqlite", True)]) +def test_actual_legacy_active_variant_still_launches( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, operation_tools: bool, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_live_fixture(tmp_path, provider=provider) + payload = json.loads(registry.read_text()) + payload["goals"][0]["status"] = "active-read-only" + registry.write_text(json.dumps(payload)) + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, operation_tools) + code, payload = _run(registry, runtime, project, host_arguments) + assert code == 0, payload + assert native_start.exists() and turn_start.exists(), payload + assert _session_files(runtime) + assert payload["effects"]["host_invoked"] is True, payload + if operation_tools: + # The shared fake operation server returns a wait, never an outcome. + _assert_no_settlement(runtime, payload) + else: + assert payload["status"] == "committed" and payload["effects"]["quota_spent"], payload + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_operation_launch_releases_registry_locks_before_native_response_wait( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, source = _write_live_fixture(tmp_path, provider=provider) + registry = _mirror(source, runtime) + host_arguments, native_start, turn_start = _host(tmp_path, monkeypatch, True) + original_write = codex_operation_host.CodexChatAgentSession._write + stopped = [] + + def stop_during_response_wait(): + stopped.append(activation_service.set_goal_activation_state( + registry_path=registry, runtime_root_override=str(runtime), goal_id=GOAL, + state="stopped", reason="Synthetic owner Stop after native Turn dispatch.", + actor_kind="owner", execute=True, + )) + + def observed_write(session, packet, **options): + original_write(session, packet, **options) + if packet.get("method") == "turn/start": + # The RPC response has not been read yet. A lifecycle writer must + # already be able to acquire both source and mirror writer locks. + writer = threading.Thread(target=stop_during_response_wait, daemon=True) + writer.start() + writer.join(timeout=5) + assert not writer.is_alive(), "native response wait retained registry locks" + assert stopped[0]["readback"]["verified"] is True, stopped + + monkeypatch.setattr(codex_operation_host.CodexChatAgentSession, "_write", observed_write) + _code, payload = _run(registry, runtime, project, host_arguments) + assert stopped and stopped[0]["readback"]["verified"] is True, payload + assert native_start.exists() and turn_start.exists(), payload + _assert_no_settlement(runtime, payload) diff --git a/tests/test_loopx_turn_codex_cli.py b/tests/test_loopx_turn_codex_cli.py index 89bfed1076..14afcd7a4f 100644 --- a/tests/test_loopx_turn_codex_cli.py +++ b/tests/test_loopx_turn_codex_cli.py @@ -693,8 +693,7 @@ def test_write_resume_rejects_unapproved_or_other_profile_changes(tmp_path, monk registry = tmp_path / "registry.json" scopes = {"file": ["worker/result.json"], "sibling": ["other/**"], "parent": ["worker/../**"]}.get(case, ["worker/**"]) - _write_resume_registry(registry, project, scopes=scopes, expired=case == "expired", - active=case != "inactive") + _write_resume_registry(registry, project, scopes=scopes, expired=case == "expired") if case in {"unregistered", "projection"}: payload = json.loads(registry.read_text()) coordination = payload["goals"][0]["coordination"] @@ -709,6 +708,8 @@ def test_write_resume_rejects_unapproved_or_other_profile_changes(tmp_path, monk options = dict(runtime_root=tmp_path / "runtime", project=worker, codex_bin=str(executable), registry_path=None if case == "missing" else registry, model="fixture-model") run_codex_cli_host(request, sandbox="read-only", **options) + if case == "inactive": + _write_resume_registry(registry, project, scopes=scopes, active=False) request["session"]["action"] = "resume" if case == "model": options["model"] = "another-model" @@ -790,6 +791,143 @@ def test_stopped_source_goal_rejects_resume_before_provider_launch( assert binding.read_bytes() == original_binding +@pytest.mark.parametrize("context", ["fresh", "resume"]) +def test_executable_legacy_goal_stop_after_selection_prevents_host_launch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, context: str, +) -> None: + from loopx.control_plane.projects.registry_codec import project_registry_transaction + from loopx.control_plane.turn_driver import codex_cli + + executable, log = _fake_codex(tmp_path) + monkeypatch.setenv("FAKE_CODEX_LOG", str(log)) + registry_path = tmp_path / "registry.json" + _write_resume_registry(registry_path, tmp_path, scopes=["**"]) + request = _request() + request["session"]["context_policy"] = {"mode": "resume", "binding_scope": "agent"} + options = {"runtime_root": tmp_path / "runtime", "project": tmp_path, + "codex_bin": str(executable), "registry_path": registry_path} + run_codex_cli_host(request, **options) + binding = _session_files(tmp_path / "runtime")[0] + original_binding = binding.read_bytes() + original_prompt = codex_cli._prompt + + def stop_before_dispatch(current): + with project_registry_transaction(registry_path, operation="fixture_stop_before_launch") as transaction: + registry = transaction.payload_copy() + registry["goals"][0]["activation"] = { + "schema_version": "loopx_goal_activation_v1", "state": "stopped", + "updated_at": "2026-10-10T12:00:00Z", "reason": "Owner stopped this Goal", + } + transaction.commit(registry) + return original_prompt(current) + + monkeypatch.setattr(codex_cli, "_prompt", stop_before_dispatch) + request["session"]["context_policy"]["mode"] = context + request["session"]["action"] = "resume" if context == "resume" else "start_new" + with pytest.raises(FirstPartyHostRuntimeRejected, match="goal_stopped"): + run_codex_cli_host(request, **options) + assert len(log.read_text().splitlines()) == 1 + assert binding.read_bytes() == original_binding + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_default_turn_cli_blocks_native_launch_after_canonical_goal_stop( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + import contextlib + import io + + from loopx.cli import main as cli_main + from loopx.control_plane.projects.registry_codec import project_registry_transaction + from loopx.control_plane.turn_driver import codex_cli + from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime + from tests.test_loopx_turn_driver import _write_live_fixture + + if provider == "sqlite": + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry_path = _write_live_fixture(tmp_path, provider=provider) + executable, log = _fake_codex(tmp_path) + monkeypatch.setenv("FAKE_CODEX_LOG", str(log)) + original_prompt = codex_cli._prompt + reached_host = [] + + def stop_before_dispatch(current): + reached_host.append(current["turn_key"]) + with project_registry_transaction(registry_path, operation="fixture_stop_before_launch") as transaction: + registry = transaction.payload_copy() + registry["goals"][0]["activation"] = { + "schema_version": "loopx_goal_activation_v1", "state": "stopped", + "updated_at": "2026-10-10T12:00:00Z", "reason": "Owner stopped this Goal", + } + transaction.commit(registry) + return original_prompt(current) + + monkeypatch.setattr(codex_cli, "_prompt", stop_before_dispatch) + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = cli_main([ + "--registry", str(registry_path), "--runtime-root", str(runtime), "--format", "json", + "turn", "run-once", "--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture", + "--host", "codex-cli", "--project", str(project), "--scan-root", str(project), + "--codex-bin", str(executable), "--codex-sandbox", "read-only", "--no-global-sync", + "--iteration-context", "fresh", "--turn-instance-id", "cancel-before-native-launch", + "--validation-command-json", json.dumps([sys.executable, "-c", "pass"]), "--execute", + ]) + payload = json.loads(output.getvalue()) + assert reached_host and code == 1, payload + assert "goal_stopped" in json.dumps(payload) + assert not log.exists() and not _session_files(runtime) + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + rows = [json.loads(line) for line in index.read_text().splitlines()] if index.exists() else [] + assert not any(row.get("classification") == "quota_slot_spent" for row in rows) + + +@pytest.mark.parametrize("context", ["fresh", "resume"]) +@pytest.mark.parametrize("change", ["stop", "replace"]) +def test_source_goal_change_after_session_selection_prevents_host_launch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, context: str, change: str, +) -> None: + from loopx.control_plane.turn_driver import codex_cli + + executable, log = _fake_codex(tmp_path) + monkeypatch.setenv("FAKE_CODEX_LOG", str(log)) + admission = _source_admission(tmp_path) + request = _request() + request["goal_ref"] = SOURCE_GOAL_REF + request["session"]["context_policy"] = {"mode": "resume", "binding_scope": "agent"} + options = {"runtime_root": tmp_path / "runtime", "project": tmp_path, + "codex_bin": str(executable), "goal_admission": admission} + run_codex_cli_host(request, **options) + binding = _session_files(tmp_path / "runtime")[0] + original_binding = binding.read_bytes() + original_prompt = codex_cli._prompt + + def change_before_dispatch(current): + # Session selection has succeeded, but no new Host request exists yet. + with source_session_registry_transaction( + admission.registry_path, operation="fixture_change_before_dispatch", + ) as transaction: + registry = transaction.payload_copy() + if change == "replace": + registry["goals"][0]["goal_instance_id"] = "ginst_" + "b" * 32 + else: + registry["goals"][0]["activation"] = { + "schema_version": "loopx_goal_activation_v1", "state": "stopped", + "updated_at": "2026-10-10T12:00:00Z", "reason": "Owner stopped this Goal", + } + transaction.commit(registry) + return original_prompt(current) + + monkeypatch.setattr(codex_cli, "_prompt", change_before_dispatch) + request["session"]["context_policy"]["mode"] = context + request["session"]["action"] = "resume" if context == "resume" else "start_new" + with pytest.raises(FirstPartyHostRuntimeRejected) as error: + run_codex_cli_host(request, **options) + assert error.value.code == ("goal_stopped" if change == "stop" else "stale_goal_instance") + assert len(log.read_text().splitlines()) == 1 + assert binding.read_bytes() == original_binding + + def test_codex_source_session_descriptor_persists_exact_goal_ref( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1420,3 +1558,76 @@ def test_codex_cli_reaps_descendants_after_result_or_timeout( os.kill(int(pid_path.read_text()), signal.SIGKILL) except ProcessLookupError: pass + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_default_turn_cli_rejects_stopped_source_behind_stale_global_mirror( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +) -> None: + import contextlib + import io + + from loopx.cli import main as cli_main + from loopx.control_plane.goals import activation_service + from loopx.control_plane.turn_driver import codex_cli + from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime + from tests.test_loopx_turn_driver import _write_live_fixture + + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, source_registry = _write_live_fixture(tmp_path, provider=provider) + mirror = runtime / "registry.global.json" + mirrored = json.loads(source_registry.read_text()) + mirrored["registry_role"] = "global-local" + mirrored["goals"][0]["source_registry"] = str(source_registry) + mirror.write_text(json.dumps(mirrored)) + original_mirror = mirror.read_bytes() + executable, native_log = _fake_codex(tmp_path) + native_launch = tmp_path / "native-launch.txt" + native_source = executable.read_text() + assert "args = sys.argv[1:]" in native_source + executable.write_text(native_source.replace( + "args = sys.argv[1:]", + f"pathlib.Path({str(native_launch)!r}).write_text('started\\n')\nargs = sys.argv[1:]", + 1, + )) + monkeypatch.setenv("FAKE_CODEX_LOG", str(native_log)) + original_prompt = codex_cli._prompt + stopped = [] + + def stop_source_before_dispatch(request): + with monkeypatch.context() as fault: + fault.setattr(activation_service, "sync_project_registry_to_global", lambda **_kwargs: { + "ok": False, "error": "fixture lost global mirror update", + }) + result = activation_service.set_goal_activation_state( + registry_path=mirror, goal_id="loopx-turn-fixture", state="stopped", + reason="Owner stopped the source while mirror synchronization failed.", + actor_kind="owner", execute=True, + ) + assert result["written"] is True and result["partial_write"] is True + assert json.loads(source_registry.read_text())["goals"][0]["activation"]["state"] == "stopped" + assert mirror.read_bytes() == original_mirror + stopped.append(result) + return original_prompt(request) + + monkeypatch.setattr(codex_cli, "_prompt", stop_source_before_dispatch) + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = cli_main([ + "--registry", str(mirror), "--runtime-root", str(runtime), "--format", "json", + "turn", "run-once", "--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture", + "--host", "codex-cli", "--project", str(project), "--scan-root", str(project), + "--codex-bin", str(executable), "--codex-sandbox", "read-only", "--no-global-sync", + "--iteration-context", "fresh", "--turn-instance-id", "stale-mirror-stop-before-native", + "--validation-command-json", json.dumps([sys.executable, "-c", "pass"]), "--execute", + ]) + payload = json.loads(output.getvalue()) + # Independent process and model-boundary markers prove this rejection did + # not dispatch. The public error remains conservative because the same + # rejection type can also arise after a Host result has already arrived. + assert stopped and not native_launch.exists() and not native_log.exists(), payload + assert code == 1 and "goal_stopped" in json.dumps(payload), payload + assert payload["effects"]["host_invoked"] is not True + assert not _session_files(runtime) + index = runtime / "goals/loopx-turn-fixture/runs/index.jsonl" + assert not index.exists() or "quota_slot_spent" not in index.read_text() From c3f421545962e7bc8915d86df20e42d2ceb7073f Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 14:54:34 +0800 Subject: [PATCH 2/6] fix(host): check native stdin deadline in the actual writer Continue the existing B3 proposal #6884; do not create another split PR. A writer thread can start or finish after the absolute dispatch deadline before the caller next observes its already-set Event. Reject expired starts before writing and late completions before reporting success. Keep timely completion valid even when the caller is scheduled late. Retain the existing cancellation/reap path, original deadline and Unicode wire semantics. This does not qualify the separate process-tree cleanup budget or the full Goal writer/Stop integration. Add deterministic late-start/late-flush scheduling tests and controls for timely completion and pre-expired admission, plus real subprocess Unicode and 8 MiB pipe-backpressure cases. Isolated pinned-module Python 3.13.5 / pytest 9.0.2: before 2 failed, 4 passed; after 6 passed. py_compile and staged diff checks passed. Full repository tests, Ruff (not installed), Windows, File/SQLite Goal integration and current-head CI were not run. Source module blob verified: 4f9121420a42b6c1328b62992af11821fc1a9a83 Prepared with ChatGPT assistance. Signed-off-by: song --- loopx/extensions/process_runtime.py | 7 + tests/test_process_runtime_stdin_deadline.py | 161 +++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 tests/test_process_runtime_stdin_deadline.py diff --git a/loopx/extensions/process_runtime.py b/loopx/extensions/process_runtime.py index 4f9121420a..ceb34722cb 100644 --- a/loopx/extensions/process_runtime.py +++ b/loopx/extensions/process_runtime.py @@ -41,8 +41,15 @@ def write_process_stdin_before_deadline( def write() -> None: try: + # Starting a thread is not dispatch: it may first run after expiry. + if time.monotonic() >= deadline: + raise TimeoutError("native input dispatch deadline expired") stream.write(text) stream.flush() + # A set Event can be observed after an expired wait. Qualify the + # actual completion here, not when the caller is next scheduled. + if time.monotonic() >= deadline: + raise TimeoutError("native input dispatch deadline expired") except BaseException as exc: failures.append(exc) finally: diff --git a/tests/test_process_runtime_stdin_deadline.py b/tests/test_process_runtime_stdin_deadline.py new file mode 100644 index 0000000000..16a3c81f11 --- /dev/null +++ b/tests/test_process_runtime_stdin_deadline.py @@ -0,0 +1,161 @@ +"""Absolute-deadline races in the native stdin transport, without a model host.""" +from __future__ import annotations + +import subprocess +import sys +import threading +import time +from types import SimpleNamespace + +import pytest + +from loopx.extensions import process_runtime + + +def _scheduled_writer(monkeypatch, *, late_start=False, late_flush=False): + """Let the writer finish before the caller is scheduled to inspect its event.""" + clock = SimpleNamespace(now=0.0) + writes = [] + cancellations = [] + + class Stream: + def write(self, text): + writes.append(text) + return len(text) + + def flush(self): + if late_flush: + clock.now = 2.0 + + class ScheduledThread(threading.Thread): + def run(self): + if late_start: + clock.now = 2.0 + super().run() + + def start(self): + super().start() + self.join(timeout=2) + assert not self.is_alive(), "controlled writer did not finish" + clock.now = 3.0 + + monkeypatch.setattr(process_runtime, "time", SimpleNamespace(monotonic=lambda: clock.now)) + monkeypatch.setattr( + process_runtime, "threading", + SimpleNamespace(Event=threading.Event, Thread=ScheduledThread), + ) + return SimpleNamespace(stdin=Stream()), writes, cancellations, clock + + +def test_native_stdin_rejects_writer_starting_after_absolute_deadline(monkeypatch): + process, writes, cancellations, _ = _scheduled_writer(monkeypatch, late_start=True) + with pytest.raises(TimeoutError, match="dispatch deadline"): + process_runtime.write_process_stdin_before_deadline( + process, "late frame\n", deadline=1.0, + cancel_and_reap=lambda: cancellations.append("cancelled"), + ) + assert writes == [] + assert cancellations == ["cancelled"] + + +def test_native_stdin_rejects_flush_completed_after_absolute_deadline(monkeypatch): + process, writes, cancellations, _ = _scheduled_writer(monkeypatch, late_flush=True) + with pytest.raises(TimeoutError, match="dispatch deadline"): + process_runtime.write_process_stdin_before_deadline( + process, "partial or late frame\n", deadline=1.0, + cancel_and_reap=lambda: cancellations.append("cancelled"), + ) + assert writes == ["partial or late frame\n"] + assert cancellations == ["cancelled"] + + +def test_native_stdin_accepts_timely_completion_when_caller_resumes_late(monkeypatch): + process, writes, cancellations, clock = _scheduled_writer(monkeypatch) + process_runtime.write_process_stdin_before_deadline( + process, "timely frame\n", deadline=1.0, + cancel_and_reap=lambda: cancellations.append("cancelled"), + ) + assert clock.now == 3.0 + assert writes == ["timely frame\n"] + assert cancellations == [] + + +def test_native_stdin_expired_before_dispatch_does_not_write(monkeypatch): + process, writes, cancellations, clock = _scheduled_writer(monkeypatch) + clock.now = 2.0 + with pytest.raises(TimeoutError, match="dispatch deadline"): + process_runtime.write_process_stdin_before_deadline( + process, "not sent\n", deadline=1.0, + cancel_and_reap=lambda: cancellations.append("cancelled"), + ) + assert writes == [] + assert cancellations == [] + + +def _close_child(process): + if process.poll() is None: + process.kill() + process.wait(timeout=2) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None: + try: + stream.close() + except (BrokenPipeError, OSError): + pass + + +def test_native_stdin_large_unicode_frame_preserves_wire_text(): + process = subprocess.Popen( + [sys.executable, "-c", "import sys; sys.stdout.write(sys.stdin.readline()); sys.stdout.flush()"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding="utf-8", + ) + text = "目标保留🙂" * 32768 + "\n" + cancellations = [] + + def cancel(): + cancellations.append("cancelled") + _close_child(process) + + try: + process_runtime.write_process_stdin_before_deadline( + process, text, deadline=time.monotonic() + 5, + cancel_and_reap=cancel, + ) + stdout, stderr = process.communicate(timeout=5) + assert process.returncode == 0 + assert stdout == text + assert stderr == "" + assert cancellations == [] + finally: + _close_child(process) + + +def test_native_stdin_backpressure_cancels_peer_and_joins_writer(): + process = subprocess.Popen( + [sys.executable, "-c", "import sys, time; print('ready', flush=True); time.sleep(60)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding="utf-8", + ) + cancellations = [] + + def cancel(): + cancellations.append("cancelled") + process.kill() + process.wait(timeout=2) + + try: + assert process.stdout.readline() == "ready\n" + with pytest.raises(TimeoutError, match="dispatch deadline"): + process_runtime.write_process_stdin_before_deadline( + process, "x" * (8 * 1024 * 1024) + "\n", + deadline=time.monotonic() + 0.05, cancel_and_reap=cancel, + ) + assert process.poll() is not None + assert cancellations == ["cancelled"] + assert not any( + thread.name == "loopx-native-stdin-writer" and thread.is_alive() + for thread in threading.enumerate() + ) + finally: + _close_child(process) From 25eeda784b478229ade5861d95457f9f77b639be Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 15:34:23 +0800 Subject: [PATCH 3/6] fix(host): complete bounded stdin retirement on current writer semantics Preserve the original RPC/Turn budget across native stdin write, flush and response. Reject partial text writes using character counts. Bound forced cleanup of the original receiver and its Windows taskkill or POSIX observer, then confirm receiver reap and writer exit before successful cancellation. Retain the current source's actual-writer start/completion deadline checks, including timely completion observed by a caller scheduled later. Supply the existing deadline test fake with the reaped-process poll contract. Preserve ordinary graceful cleanup and the existing launch-admission extraction. Cleanup refusal remains an explicit unconfirmed failure, not proof of process retirement. This change does not make every whole-host close path bounded. Co-authored-by: Codex Signed-off-by: Codex Signed-off-by: song --- loopx/chat_agent.py | 23 +- loopx/extensions/process_runtime.py | 147 +++++++++- tests/extensions/test_process_runtime.py | 293 +++++++++++++++++++ tests/test_chat_agent.py | 90 ++++++ tests/test_process_runtime_stdin_deadline.py | 2 +- 5 files changed, 528 insertions(+), 27 deletions(-) diff --git a/loopx/chat_agent.py b/loopx/chat_agent.py index fae30630a3..227d3724b2 100644 --- a/loopx/chat_agent.py +++ b/loopx/chat_agent.py @@ -1123,28 +1123,29 @@ def _guarded_write( self, frame: str, write_guard: Callable[[], AbstractContextManager[None]], *, deadline: float | None, ) -> None: - from .extensions.process_runtime import write_process_stdin_before_deadline - - # One frame has at most one second under the Goal's launch fence. - # A shorter original RPC/Turn deadline still wins. Waiting for another - # native writer spends this allowance before any authority is acquired. - dispatch_deadline = min( - time.monotonic() + 1.0, - deadline if deadline is not None else time.monotonic() + self.response_timeout_sec, - ) + from .extensions.process_runtime import force_terminate_process, write_process_stdin_before_deadline + + # Serialization, delivery and response share the original RPC/Turn + # deadline. No shorter send budget may reject an otherwise valid frame. + dispatch_deadline = deadline if deadline is not None else time.monotonic() + self.response_timeout_sec if not self._write_lock.acquire(timeout=max(0, dispatch_deadline - time.monotonic())): raise self._timeout_error("response_timeout", "Codex input dispatch timed out.") try: with write_guard(): + process = self.process try: write_process_stdin_before_deadline( - self.process, frame, deadline=dispatch_deadline, - cancel_and_reap=self.close, + process, frame, deadline=dispatch_deadline, + cancel_and_reap=lambda: force_terminate_process( + process, process_tree_owned=self.process_tree_owned, + ), ) except TimeoutError as exc: raise self._timeout_error( "response_timeout", "Codex input dispatch timed out." ) from exc + except RuntimeError as exc: + raise self._runtime_error("Codex native input cleanup could not be confirmed.") from exc finally: self._write_lock.release() diff --git a/loopx/extensions/process_runtime.py b/loopx/extensions/process_runtime.py index ceb34722cb..aa74afa28e 100644 --- a/loopx/extensions/process_runtime.py +++ b/loopx/extensions/process_runtime.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +import math import os import shutil import signal @@ -44,7 +45,8 @@ def write() -> None: # Starting a thread is not dispatch: it may first run after expiry. if time.monotonic() >= deadline: raise TimeoutError("native input dispatch deadline expired") - stream.write(text) + if stream.write(text) != len(text): + raise OSError("incomplete native input frame") stream.flush() # A set Event can be observed after an expired wait. Qualify the # actual completion here, not when the caller is next scheduled. @@ -62,15 +64,25 @@ def write() -> None: raise TimeoutError("native input dispatch deadline expired") if failures: raise failures[0] + writer.join(timeout=max(0, deadline - time.monotonic())) + if writer.is_alive(): + raise TimeoutError("native input dispatch deadline expired") except BaseException: + cleanup_error: BaseException | None = None try: cancel_and_reap() - finally: - writer.join(timeout=_PROCESS_TERMINATE_GRACE_SECONDS) + except BaseException as error: + cleanup_error = error + writer.join(timeout=_PROCESS_TERMINATE_GRACE_SECONDS) if writer.is_alive(): - raise RuntimeError("native input writer did not drain after peer cancellation") + # Closing a TextIOWrapper here can wait forever for this writer's + # stream lock. Report unconfirmed retirement without closing it. + raise RuntimeError("native input writer did not drain after peer cancellation") from cleanup_error + if cleanup_error is not None: + raise RuntimeError("native input receiver cleanup unconfirmed") from cleanup_error + if process.poll() is None: + raise RuntimeError("native input receiver was not reaped") raise - writer.join() @dataclass(frozen=True) @@ -88,11 +100,27 @@ def _wait_for_process(process: subprocess.Popen[bytes] | subprocess.Popen[str], return True -def _posix_owned_group_has_exited(process_group_id: int, timeout: float) -> bool: - snapshot = subprocess.run( +def _bounded_posix_group_snapshot(timeout: float) -> subprocess.CompletedProcess[str]: + deadline = time.monotonic() + timeout + observer = subprocess.Popen( ["ps", "-A", "-o", "pgid=", "-o", "stat="], - capture_output=True, text=True, encoding="utf-8", check=False, - timeout=timeout, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding="utf-8", + ) + try: + output, _ = observer.communicate(timeout=max(0, deadline - time.monotonic()) / 2) + return subprocess.CompletedProcess(observer.args, observer.returncode, output) + finally: + # Reserve time to kill/reap the observer too. POSIX communicate has no + # background reader that can retain the stdout lock after returning. + _kill_and_reap_before_deadline(observer, deadline) + if observer.stdout is not None: + observer.stdout.close() + + +def _posix_owned_group_has_exited(process_group_id: int, timeout: float, *, bounded_reap: bool = False) -> bool: + snapshot = _bounded_posix_group_snapshot(timeout) if bounded_reap else subprocess.run( + ["ps", "-A", "-o", "pgid=", "-o", "stat="], + capture_output=True, text=True, encoding="utf-8", check=False, timeout=timeout, ) if snapshot.returncode != 0 or not snapshot.stdout.strip(): raise RuntimeError("owned POSIX process-group observation failed") @@ -109,8 +137,10 @@ def _posix_owned_group_has_exited(process_group_id: int, timeout: float) -> bool return not live -def _wait_for_posix_process_group_stop(process_group_id: int) -> None: - deadline = time.monotonic() + _PROCESS_GROUP_STOP_TIMEOUT_SECONDS +def _wait_for_posix_process_group_stop(process_group_id: int, *, deadline: float | None = None) -> None: + bounded_reap = deadline is not None + observation_deadline = time.monotonic() + _PROCESS_GROUP_STOP_TIMEOUT_SECONDS + deadline = min(deadline, observation_deadline) if deadline is not None else observation_deadline while True: try: os.killpg(process_group_id, 0) @@ -124,7 +154,9 @@ def _wait_for_posix_process_group_stop(process_group_id: int) -> None: if remaining <= 0: raise TimeoutError("owned POSIX process group did not stop before cleanup deadline") try: - exited = _posix_owned_group_has_exited(process_group_id, remaining) + exited = _posix_owned_group_has_exited( + process_group_id, remaining, **({"bounded_reap": True} if bounded_reap else {}), + ) except (OSError, subprocess.TimeoutExpired, UnicodeError) as error: raise RuntimeError("owned POSIX process-group observation failed") from error if exited: @@ -132,13 +164,18 @@ def _wait_for_posix_process_group_stop(process_group_id: int) -> None: time.sleep(min(.01, max(0, deadline - time.monotonic()))) -def _darwin_owned_group_has_exited(process: subprocess.Popen[bytes] | subprocess.Popen[str]) -> bool: +def _darwin_owned_group_has_exited( + process: subprocess.Popen[bytes] | subprocess.Popen[str], *, timeout: float = 1, + bounded_reap: bool = False, +) -> bool: # Darwin can report EPERM rather than ESRCH for a now-empty process group. # A reaped leader alone does not prove its descendants have exited. - if sys.platform != "darwin" or process.poll() is None: + if sys.platform != "darwin" or process.poll() is None or timeout <= 0: return False try: - return _posix_owned_group_has_exited(process.pid, 1) + return _posix_owned_group_has_exited( + process.pid, min(1, timeout), **({"bounded_reap": True} if bounded_reap else {}), + ) except (OSError, subprocess.TimeoutExpired, UnicodeError, RuntimeError): return False @@ -227,6 +264,86 @@ def terminate_process_tree( process.wait() +def _kill_and_reap_before_deadline( + process: subprocess.Popen[bytes] | subprocess.Popen[str], deadline: float, +) -> None: + try: + if process.poll() is None: + process.kill() + except ProcessLookupError: + pass + finally: + process.wait(timeout=max(0, deadline - time.monotonic())) + + +def _force_terminate_posix_process_group( + process: subprocess.Popen[bytes] | subprocess.Popen[str], deadline: float, +) -> None: + try: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + except PermissionError: + if not _darwin_owned_group_has_exited( + process, timeout=deadline - time.monotonic(), bounded_reap=True, + ): + raise + return + finally: + # Even a failed group signal must retire the original pipe receiver. + _kill_and_reap_before_deadline(process, deadline) + _wait_for_posix_process_group_stop(process.pid, deadline=deadline) + + +def _force_terminate_windows_process_tree( + process: subprocess.Popen[bytes] | subprocess.Popen[str], deadline: float, +) -> None: + if process.poll() is not None: + process.wait(timeout=max(0, deadline - time.monotonic())) + # This owner did not retain a Job Object. An exited leader alone cannot + # establish that descendants holding the pipe also stopped. + raise RuntimeError("owned Windows process-tree cleanup unconfirmed after leader exit") + try: + taskkill = subprocess.Popen( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + if taskkill.wait(timeout=max(0, deadline - time.monotonic()) / 2) != 0: + raise RuntimeError("owned Windows process-tree cleanup unconfirmed") + finally: + # subprocess.run(timeout=...) internally waits without a timeout + # after killing its helper. Bound that helper's own reap as well. + helper_deadline = time.monotonic() + max(0, deadline - time.monotonic()) / 2 + _kill_and_reap_before_deadline(taskkill, helper_deadline) + finally: + # Reserve the remaining budget for this Popen handle even if taskkill + # is unavailable, times out, or cannot prove the descendant tree stopped. + _kill_and_reap_before_deadline(process, deadline) + + +def force_terminate_process( + process: subprocess.Popen[bytes] | subprocess.Popen[str], *, + process_tree_owned: bool, timeout: float = _PROCESS_TERMINATE_GRACE_SECONDS, +) -> None: + """Bound failed native-send retirement; ordinary graceful cleanup is unchanged. + + A caller holding a launch fence must keep it until this original receiver + is reaped and the stdin writer exits. Cleanup has a separate bounded budget + after the original request expires; it never grants more time to send. + """ + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("native process cleanup timeout must be positive and finite") + deadline = time.monotonic() + timeout + if process_tree_owned and os.name == "posix": + _force_terminate_posix_process_group(process, deadline) + elif process_tree_owned and os.name == "nt": + _force_terminate_windows_process_tree(process, deadline) + else: + _kill_and_reap_before_deadline(process, deadline) + + def prepare_owned_process_cleanup(process: subprocess.Popen[bytes]) -> Callable[[], None]: """Retain diagnostic tree ownership even if its leader exits. diff --git a/tests/extensions/test_process_runtime.py b/tests/extensions/test_process_runtime.py index eeee1505c0..cdbe430a8b 100644 --- a/tests/extensions/test_process_runtime.py +++ b/tests/extensions/test_process_runtime.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor import json import os from pathlib import Path @@ -7,6 +8,7 @@ import stat import subprocess import sys +import threading import time from unittest.mock import Mock @@ -14,12 +16,303 @@ import loopx.extensions.process_runtime as process_runtime from loopx.extensions.process_runtime import ( + force_terminate_process, remove_owned_tree, run_capped_process, terminate_process_tree, + write_process_stdin_before_deadline, ) +@pytest.mark.parametrize("written", [None, 0, 2]) +def test_partial_native_frame_retires_receiver_before_return(written) -> None: + process = Mock(spec=subprocess.Popen) + process.stdin = Mock() + process.stdin.write.return_value = written + process.poll.return_value = None + retired = [] + + def cancel() -> None: + retired.append(True) + process.poll.return_value = -9 + + with pytest.raises(OSError, match="incomplete native input frame"): + write_process_stdin_before_deadline( + process, "完整☃\n", deadline=time.monotonic() + 2, cancel_and_reap=cancel, + ) + assert retired == [True] and process.poll() is not None + process.stdin.flush.assert_not_called() + process.stdin.close.assert_not_called() + + +def test_complete_native_unicode_frame_uses_character_count() -> None: + process = Mock(spec=subprocess.Popen) + process.stdin = Mock() + text = "完整☃\n" + process.stdin.write.return_value = len(text) + cancel = Mock() + write_process_stdin_before_deadline( + process, text, deadline=time.monotonic() + 2, cancel_and_reap=cancel, + ) + process.stdin.write.assert_called_once_with(text) + process.stdin.flush.assert_called_once_with() + process.stdin.close.assert_not_called() + cancel.assert_not_called() + + +def test_failed_native_cancellation_still_observes_blocked_writer() -> None: + process = Mock(spec=subprocess.Popen) + process.stdin = Mock() + release = threading.Event() + writers = [] + + def blocked_write(text): + writers.append(threading.current_thread()) + assert release.wait(timeout=5) + return len(text) + + process.stdin.write.side_effect = blocked_write + failure = OSError("synthetic native cleanup failure") + try: + began = time.monotonic() + with pytest.raises(RuntimeError, match="writer did not drain") as raised: + write_process_stdin_before_deadline( + process, "frame\n", deadline=began + .1, + cancel_and_reap=Mock(side_effect=failure), + ) + assert raised.value.__cause__ is failure + assert time.monotonic() - began < 2.5 + process.stdin.close.assert_not_called() + finally: + release.set() + for writer in writers: + writer.join(timeout=2) + assert not writer.is_alive() + + +@pytest.mark.parametrize("phase", ["write", "flush"]) +def test_native_stream_error_retires_receiver_before_propagation(phase) -> None: + process = Mock(spec=subprocess.Popen) + process.stdin = Mock() + process.stdin.write.return_value = len("frame\n") + failure = OSError("synthetic partial native stream error") + getattr(process.stdin, phase).side_effect = failure + process.poll.return_value = None + cancel = Mock(side_effect=lambda: setattr(process.poll, "return_value", -9)) + with pytest.raises(OSError) as raised: + write_process_stdin_before_deadline( + process, "frame\n", deadline=time.monotonic() + 2, cancel_and_reap=cancel, + ) + assert raised.value is failure and process.poll() is not None + cancel.assert_called_once_with() + process.stdin.close.assert_not_called() + + +@pytest.mark.parametrize("outcome", ["success", "failed", "timeout", "missing", "unreaped_helper"]) +def test_windows_force_cleanup_reserves_bounded_original_receiver_reap(monkeypatch, outcome): + process = Mock(spec=subprocess.Popen) + helper = Mock(spec=subprocess.Popen) + process.pid, process.poll.return_value = 12345, None + helper.poll.return_value = None + clock, events = [0.0], [] + monkeypatch.setattr(time, "monotonic", lambda: clock[0]) + + def taskkill(command, **kwargs): + assert command == ["taskkill", "/PID", "12345", "/T", "/F"] + assert kwargs == {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL} + events.append("tree-start") + if outcome == "missing": + raise FileNotFoundError("taskkill unavailable") + return helper + + def helper_wait(*, timeout): + assert 0 < timeout <= .5 + first = "tree-wait" not in events + events.append("tree-wait" if first else "tree-reap") + clock[0] += timeout + if outcome == "unreaped_helper" or (first and outcome == "timeout"): + raise subprocess.TimeoutExpired("taskkill", timeout) + helper.poll.return_value = int(outcome == "failed") + return helper.poll.return_value + + def reap(*, timeout): + assert 0 < timeout <= (1 if outcome == "missing" else .5) + events.append("reap") + process.poll.return_value = -9 + return -9 + + monkeypatch.setattr(subprocess, "Popen", taskkill) + helper.kill.side_effect = lambda: events.append("tree-kill") + helper.wait.side_effect = helper_wait + process.kill.side_effect = lambda: events.append("kill") + process.wait.side_effect = reap + if outcome == "success": + process_runtime._force_terminate_windows_process_tree(process, 1) + else: + with pytest.raises((RuntimeError, subprocess.TimeoutExpired, FileNotFoundError)): + process_runtime._force_terminate_windows_process_tree(process, 1) + assert events[-2:] == ["kill", "reap"] and process.poll() is not None + if outcome == "missing": + assert events == ["tree-start", "kill", "reap"] + else: + assert events.count("tree-start") == 1 and events.count("tree-reap") == 1 + assert ("tree-kill" in events) == (outcome in {"timeout", "unreaped_helper"}) + + +def test_windows_exited_leader_does_not_certify_descendant_cleanup(monkeypatch): + process = Mock(spec=subprocess.Popen) + process.poll.return_value = 0 + taskkill = Mock() + monkeypatch.setattr(subprocess, "Popen", taskkill) + with pytest.raises(RuntimeError, match="unconfirmed after leader exit"): + process_runtime._force_terminate_windows_process_tree(process, time.monotonic() + .1) + taskkill.assert_not_called() + process.kill.assert_not_called() + assert 0 <= process.wait.call_args.kwargs["timeout"] <= .1 + + +def test_windows_force_cleanup_cannot_accept_an_unreaped_receiver(monkeypatch): + process = Mock(spec=subprocess.Popen) + helper = Mock(spec=subprocess.Popen) + helper.wait.return_value, helper.poll.return_value = 0, 0 + process.pid, process.poll.return_value = 12345, None + monkeypatch.setattr(subprocess, "Popen", Mock(return_value=helper)) + process.wait.side_effect = subprocess.TimeoutExpired("native", .1) + with pytest.raises(subprocess.TimeoutExpired): + process_runtime._force_terminate_windows_process_tree(process, time.monotonic() + .1) + process.wait.assert_called_once() + assert 0 <= process.wait.call_args.kwargs["timeout"] <= .1 + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX process-group regression") +def test_force_cleanup_darwin_observation_spends_only_remaining_budget(monkeypatch): + process = Mock(spec=subprocess.Popen) + process.pid, process.poll.return_value = 12345, 0 + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(os, "killpg", Mock(side_effect=PermissionError("empty owned group"))) + observer = Mock(spec=subprocess.Popen) + observer.communicate.return_value = ("1 S\n12345 Z\n", None) + observer.poll.return_value, observer.returncode = 0, 0 + observer.args, observer.stdout = ["ps"], Mock() + monkeypatch.setattr(subprocess, "Popen", Mock(return_value=observer)) + force_terminate_process(process, process_tree_owned=True, timeout=.1) + assert 0 < observer.communicate.call_args.kwargs["timeout"] <= .05 + assert 0 <= observer.wait.call_args.kwargs["timeout"] <= .1 + assert 0 <= process.wait.call_args.kwargs["timeout"] <= .1 + process.kill.assert_not_called() + + +def test_force_posix_observer_timeout_has_bounded_self_reap(monkeypatch): + observer = Mock(spec=subprocess.Popen) + observer.poll.return_value, observer.stdout = None, Mock() + clock = [0.0] + monkeypatch.setattr(time, "monotonic", lambda: clock[0]) + + def collect(*, timeout): + assert 0 < timeout <= .05 + clock[0] += timeout + raise subprocess.TimeoutExpired("ps", timeout) + + observer.communicate.side_effect = collect + monkeypatch.setattr(subprocess, "Popen", Mock(return_value=observer)) + with pytest.raises(subprocess.TimeoutExpired): + process_runtime._bounded_posix_group_snapshot(.1) + observer.kill.assert_called_once_with() + assert 0 < observer.wait.call_args.kwargs["timeout"] <= .05 + observer.stdout.close.assert_called_once_with() + + +@pytest.mark.parametrize("timeout", [0, -1, float("nan"), float("inf")]) +def test_force_cleanup_rejects_an_unbounded_or_invalid_budget(timeout): + process = Mock(spec=subprocess.Popen) + with pytest.raises(ValueError, match="positive and finite"): + force_terminate_process(process, process_tree_owned=True, timeout=timeout) + process.kill.assert_not_called() + process.wait.assert_not_called() + + +@pytest.mark.parametrize("failure", ["backpressure", "short_write", "partial_error"]) +def test_native_pipe_failure_reaps_receiver_and_writer_before_fence_release(tmp_path, failure): + ready, release, accepted = (tmp_path / name for name in ("ready", "release", "accepted")) + if failure != "backpressure": + release.touch() + peer = ( + "import sys,time\nfrom pathlib import Path\n" + f"Path({str(ready)!r}).touch()\n" + f"while not Path({str(release)!r}).exists(): time.sleep(.01)\n" + "line=sys.stdin.readline()\n" + f"if line.endswith('\\n'): Path({str(accepted)!r}).write_text(line, encoding='utf-8')\n" + ) + process = subprocess.Popen( + [sys.executable, "-c", peer], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, text=True, encoding="utf-8", + start_new_session=os.name == "posix", + creationflags=getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0, + ) + stream = process.stdin + assert stream is not None + writers, retired = [], [] + fence = threading.Lock() + + class NativeInput: + def write(self, text): + writers.append(threading.current_thread()) + if failure == "backpressure": + return stream.write(text) + stream.write(text[:32]) + stream.flush() + if failure == "partial_error": + raise BrokenPipeError("synthetic failure after a real partial send") + return 32 + + def flush(self): + stream.flush() + + process.stdin = NativeInput() + + def cancel() -> None: + assert fence.locked() + force_terminate_process(process, process_tree_owned=True) + retired.append(process.returncode) + + def stop() -> int: + with fence: + assert process.poll() is not None + assert all(not writer.is_alive() for writer in writers) + return process.returncode + + try: + ready_deadline = time.monotonic() + 5 + while not ready.exists() and process.poll() is None and time.monotonic() < ready_deadline: + time.sleep(.01) + assert ready.exists() + with ThreadPoolExecutor(max_workers=1) as executor: + with fence: + stopped = executor.submit(stop) + began = time.monotonic() + expected = TimeoutError if failure == "backpressure" else OSError + with pytest.raises(expected): + write_process_stdin_before_deadline( + process, "完整☃" * (1024 * 1024) + "\n", + deadline=began + .1, cancel_and_reap=cancel, + ) + assert time.monotonic() - began < 2.5 + assert retired == [process.returncode] and process.poll() is not None + assert writers and all(not writer.is_alive() for writer in writers) + assert not stopped.done() + assert stopped.result(timeout=2) == process.returncode + assert not accepted.exists() + finally: + release.touch() + if process.poll() is None: + force_terminate_process(process, process_tree_owned=True) + for writer in writers: + writer.join(timeout=2) + assert not writer.is_alive() + stream.close() + assert not accepted.exists() + + @pytest.mark.skipif(os.name != "posix", reason="POSIX process-group regression") def test_zero_grace_sends_one_force_kill_and_reaps_leader( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_chat_agent.py b/tests/test_chat_agent.py index 6a8a0fc8e7..3189289d53 100644 --- a/tests/test_chat_agent.py +++ b/tests/test_chat_agent.py @@ -1,9 +1,13 @@ from __future__ import annotations +import contextlib import io import json +import os import queue import stat +import subprocess +import sys import threading import time from types import SimpleNamespace @@ -72,6 +76,92 @@ def kill(self) -> None: self.returncode = -1 +@pytest.mark.parametrize("phase", ["write", "flush"]) +def test_guarded_native_request_keeps_original_send_and_response_budget(tmp_path, phase): + process = _FakeAppServerProcess() + session = chat_agent.CodexChatAgentSession( + process=process, messages=queue.Queue(), thread_id="original-thread", work_dir=tmp_path, + response_timeout_sec=3, _launch_guard=contextlib.nullcontext, + ) + release = threading.Event() + + class SlowInput(io.StringIO): + def write(self, text): + if phase == "write": + assert release.wait(timeout=4) + return super().write(text) + + def flush(self): + if phase == "flush": + assert release.wait(timeout=4) + packet = json.loads(self.getvalue()) + session.messages.put({"id": packet["id"], "result": {"threadId": "original-thread"}}) + + process.stdin = SlowInput() + timer = threading.Timer(1.2, release.set) + timer.start() + try: + result = session._request("turn/start", {"threadId": "original-thread", "text": "完整☃"}) + assert result == {"threadId": "original-thread"} + assert process.poll() is None + assert json.loads(process.stdin.getvalue())["params"]["text"] == "完整☃" + assert session._response_waiters == {} + finally: + release.set() + timer.join(timeout=2) + + +def test_slow_native_reader_accepts_complete_unicode_with_original_deadline(tmp_path): + ready = tmp_path / "ready" + peer = ( + "import json,sys,time\nfrom pathlib import Path\n" + f"Path({str(ready)!r}).touch()\n" + "time.sleep(1.2)\nrow=json.loads(sys.stdin.readline())\n" + "text=row['params']['text']\n" + "print(json.dumps({'id':row['id'],'result':{'threadId':row['params']['threadId']," + "'complete':text == '完整☃' * (1024 * 1024)}}),flush=True)\n" + "sys.stdin.read()\n" + ) + process = subprocess.Popen( + [sys.executable, "-c", peer], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, encoding="utf-8", start_new_session=os.name == "posix", + creationflags=getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0, + ) + session = chat_agent.CodexChatAgentSession( + process=process, messages=queue.Queue(), thread_id="original-thread", work_dir=tmp_path, + response_timeout_sec=3, process_tree_owned=True, _launch_guard=contextlib.nullcontext, + ) + + def read_response(): + line = process.stdout.readline() + session.messages.put(json.loads(line) if line else EOFError()) + + reader = threading.Thread(target=read_response, daemon=True) + reader.start() + try: + deadline = time.monotonic() + 5 + while not ready.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(.01) + assert ready.exists() + result = session._request("turn/start", { + "threadId": "original-thread", "text": "完整☃" * (1024 * 1024), + }) + assert result == {"threadId": "original-thread", "complete": True} + assert process.poll() is None and session._response_waiters == {} + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=2) + reader.join(timeout=2) + assert not reader.is_alive() + for writer in threading.enumerate(): + if writer.name == "loopx-native-stdin-writer": + writer.join(timeout=2) + assert not writer.is_alive() + process.stdin.close() + process.stdout.close() + + @pytest.mark.parametrize("resume", [False, True]) def test_managed_chat_materializes_only_new_threads_without_a_turn(monkeypatch, tmp_path, resume): process = _FakeAppServerProcess() diff --git a/tests/test_process_runtime_stdin_deadline.py b/tests/test_process_runtime_stdin_deadline.py index 16a3c81f11..82c2b93ce5 100644 --- a/tests/test_process_runtime_stdin_deadline.py +++ b/tests/test_process_runtime_stdin_deadline.py @@ -44,7 +44,7 @@ def start(self): process_runtime, "threading", SimpleNamespace(Event=threading.Event, Thread=ScheduledThread), ) - return SimpleNamespace(stdin=Stream()), writes, cancellations, clock + return SimpleNamespace(stdin=Stream(), poll=lambda: 0), writes, cancellations, clock def test_native_stdin_rejects_writer_starting_after_absolute_deadline(monkeypatch): From 2591944d041c24bba75c1f0f483b1c013c47b57e Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 19:14:47 +0800 Subject: [PATCH 4/6] fix(host): bound ordinary bridge launch handoff before releasing admission Signed-off-by: song --- benchmark/tests/test_resume_sessions.py | 6 +- .../integrations/runtime-connector-catalog.md | 25 ++- .../turn_driver/host_bridge_transport.py | 99 +++++++++ .../turn_driver/host_process_bridge.ts | 26 ++- .../turn_driver/host_process_transport.py | 192 +++++++++++------- tests/test_host_launch_authority.py | 129 ++++++++++++ 6 files changed, 392 insertions(+), 85 deletions(-) create mode 100644 loopx/control_plane/turn_driver/host_bridge_transport.py diff --git a/benchmark/tests/test_resume_sessions.py b/benchmark/tests/test_resume_sessions.py index 67562e6aa2..bf3333a4c1 100644 --- a/benchmark/tests/test_resume_sessions.py +++ b/benchmark/tests/test_resume_sessions.py @@ -229,8 +229,10 @@ def test_timeout_keeps_the_observed_session_for_next_heartbeat(tmp_path, monkeyp observed_sessions = [] @contextmanager - def timeout_after_session_event(command, *, env, stdout, stderr): - with real_child_process(command, env=env, stdout=stdout, stderr=stderr) as process: + def timeout_after_session_event(command, *, env, stdout, stderr, goal_admission=None, + check_current_launch=None): + with real_child_process(command, env=env, stdout=stdout, stderr=stderr, + goal_admission=goal_admission, check_current_launch=check_current_launch) as process: def communicate(*, input, timeout): assert timeout == 1 # Inject the timeout after the real child publishes its native diff --git a/docs/integrations/runtime-connector-catalog.md b/docs/integrations/runtime-connector-catalog.md index 529a071e91..d726eb3fa0 100644 --- a/docs/integrations/runtime-connector-catalog.md +++ b/docs/integrations/runtime-connector-catalog.md @@ -99,15 +99,23 @@ lock covers each launch/send boundary and is released before waiting for a provider response. A Goal stopped after session selection therefore cannot start a new Host or replace its saved session with an empty, unlaunched result. -For operation-host `turn/start`, native stdin delivery has a one-second ceiling -and also respects any shorter original response/idle/hard deadline. Waiting -for another native writer consumes that allowance before acquiring the Goal -fence. Backpressure or an interrupted partial send cancels and reaps the same +For operation-host `turn/start`, native stdin delivery and response share the +original response/idle/hard deadline. Waiting for another native writer consumes +that allowance before acquiring the Goal fence; there is no separate one-second +ceiling on a healthy send. Backpressure or an interrupted partial send cancels and reaps the same native peer and drains the writer before releasing the fence; it cannot leave a sender to deliver that Turn after Stop. This is a dispatch timeout, not proof of non-acceptance or permission to replay. Successful large and Unicode inputs retain the full original text stream; no new input-size truncation is applied. +The guarded ordinary-Host bridge also bounds readiness, request delivery and +spawn observation within the original explicit command deadline. Without a +command deadline, only this handoff has a 30-second safety budget; a successfully +spawned Host still runs to natural completion. Cancellation retains the Goal +fence until the same bridge reports no launch or completes its Host cleanup and +is reaped. Missing cleanup evidence is an explicit supervision failure, not a +certificate that an unobserved Host stopped or permission to replay. + Source-session admission retains its exact Goal lifetime checks. Executable legacy registries instead compare their original registry byte witness under the existing registry writer lock and reuse the registry owner's legal active @@ -127,11 +135,16 @@ Codex operation tools 路径在创建 app-server 和发送 `turn/start` 前复 准入检查;只在这两个边界持有原 owner 的锁,等待模型响应前释放。这样 Stop 既不会被另一条 Host 路径绕过,也不会被长时间持锁拖延。legacy 的合法 `active-*` 状态复用原注册表规则,source-session 的生命周期校验保持严格。 -`turn/start` 的原生 stdin 发送最多占用一秒,并遵守更短的原 response、idle、hard -期限;等待其他原生写入也计入该界限。背压或部分发送中断时,先取消并回收同一个 +`turn/start` 的原生 stdin 发送与响应共享原 response、idle、hard 期限;等待其他 +原生写入也计入该界限,不再对正常发送额外施加一秒上限。背压或部分发送中断时,先取消并回收同一个 原生进程、等待写入线程退出,再释放 Goal 锁,避免 Stop 后迟到派发。这类超时不 证明请求从未受理,也不授权重放;正常的大输入和 Unicode 文本完整保留,不截断。 +普通 Host 的受控 bridge 也在原显式命令期限内完成 ready、请求发送和 spawn 观察。 +没有命令期限时,仅交接阶段有 30 秒安全预算;成功启动后仍等待 Host 自然结束。 +取消时继续持有 Goal 锁,直到同一个 bridge 报告未启动,或完成 Host 清理并被回收。 +缺少清理证据会明确报监督失败,不能据此证明未观察到的 Host 已停止,也不授权重放。 + The executable `turn run-once` path also reruns its existing current quota and health decision under those same locks, against the actual source registry and the original Turn/Todo/replan selection. It creates no replacement receipt. diff --git a/loopx/control_plane/turn_driver/host_bridge_transport.py b/loopx/control_plane/turn_driver/host_bridge_transport.py new file mode 100644 index 0000000000..61cb305bbc --- /dev/null +++ b/loopx/control_plane/turn_driver/host_bridge_transport.py @@ -0,0 +1,99 @@ +"""Bound the private bridge handoff; TypeScript still owns Host cleanup.""" +from __future__ import annotations + +import json +import os +import queue +import signal +import subprocess +import threading +import time +from typing import Any + +from ...extensions.process_runtime import force_terminate_process, write_process_stdin_before_deadline + + +class GuardedHostBridge: + """Keep at most one transient output frame while stdin is backpressured.""" + + def __init__(self, process: subprocess.Popen[str], *, deadline: float, cleanup_timeout: float) -> None: + assert process.stdout is not None + self.output = process.stdout + self.process, self.deadline, self.cleanup_timeout = process, deadline, cleanup_timeout + self.pending: queue.Queue[str | BaseException | None] = queue.Queue(maxsize=1) + self.requested = self.cancelled = self.not_launched = False + self.result: dict[str, Any] | None = None + self.reader = threading.Thread(target=self._read, name="loopx-host-bridge-reader", daemon=True) + self.reader.start() + + def _read(self) -> None: + try: + for line in self.output: + self.pending.put(line) + except BaseException as error: + self.pending.put(error) + finally: + self.pending.put(None) + + def next(self, *, deadline: float | None = None) -> dict[str, Any] | None: + try: + item = self.pending.get(timeout=None if deadline is None else max(0, deadline - time.monotonic())) + except queue.Empty: + raise TimeoutError("managed Host bridge launch deadline expired") from None + if isinstance(item, BaseException): + raise item + if item is None: + return None + event = json.loads(item) + if not isinstance(event, dict): + raise RuntimeError("invalid managed Host bridge observation") + return event + + def send(self, frame: str) -> None: + if self.next(deadline=self.deadline) != {"kind": "ready"}: + raise RuntimeError("managed Host bridge did not become ready") + self.requested = True + write_process_stdin_before_deadline(self.process, frame, deadline=self.deadline, + cancel_and_reap=self.cancel) + + def cancel(self) -> None: + if self.cancelled: + return + self.cancelled = True + deadline = time.monotonic() + self.cleanup_timeout + try: + if not self.requested: + # No request bytes have been sent, so this bridge cannot own a + # Host. This also handles suspension before JS handlers exist. + force_terminate_process(self.process, process_tree_owned=True) + self.not_launched = True + elif self.process.poll() is None: + self.process.terminate() + if os.name == "posix": + # An owned suspended bridge must run its abort handler and + # report cleanup before the Goal fence can be released. + try: + os.kill(self.process.pid, signal.SIGCONT) + except ProcessLookupError: + pass + while (event := self.next(deadline=deadline)) is not None: + if event.get("kind") == "not_launched": + self.not_launched = True + elif event.get("kind") == "result": + self.result = event + self.process.wait(timeout=max(0, deadline - time.monotonic())) + self.reader.join(timeout=max(0, deadline - time.monotonic())) + if (self.reader.is_alive() or (self.requested and + (self.process.returncode != 0 or not (self.not_launched or self.result)))): + raise RuntimeError("managed Host bridge cleanup unconfirmed") + except BaseException as error: + # Killing the bridge alone cannot certify an unobserved detached + # Host. Retain the launching record and fail instead of claiming it. + if self.process.poll() is None: + force_terminate_process(self.process, process_tree_owned=True) + raise RuntimeError("managed Host bridge cleanup unconfirmed") from error + + def close_output(self) -> None: + self.reader.join(timeout=self.cleanup_timeout) + if not self.reader.is_alive(): + self.output.close() diff --git a/loopx/control_plane/turn_driver/host_process_bridge.ts b/loopx/control_plane/turn_driver/host_process_bridge.ts index 96091020fe..ef1f97b78e 100644 --- a/loopx/control_plane/turn_driver/host_process_bridge.ts +++ b/loopx/control_plane/turn_driver/host_process_bridge.ts @@ -2,9 +2,20 @@ import {once} from "node:events"; import {decodeHostProcessRequest, runHostProcess} from "./host_process.ts"; const owner = new AbortController(); -process.stdin.on("end", () => owner.abort()); -process.on("SIGTERM", () => owner.abort()); -process.on("SIGINT", () => owner.abort()); +// Private transport observations; none grants Goal or execution authority. +type LaunchObservation = {kind: "ready"} | {kind: "not_launched"}; +const launchDeadline = process.argv[2] === "--guarded-launch" ? Number(process.argv[3]) : null; +if (launchDeadline !== null && (!Number.isFinite(launchDeadline) || launchDeadline <= 0)) + throw new TypeError("invalid guarded Host bridge deadline"); +const abort = () => { + owner.abort(); + if (!accepted) { + void emit({kind: "not_launched"} satisfies LaunchObservation).finally(() => process.stdin.destroy()); + } +}; +process.stdin.on("end", abort); +process.on("SIGTERM", abort); +process.on("SIGINT", abort); process.stdout.on("error", () => owner.abort()); let pending = Buffer.alloc(0), accepted = false; const emit = async (item: unknown) => { @@ -22,6 +33,14 @@ process.stdin.on("data", (chunk: Buffer) => { void (async () => { try { const raw = JSON.parse(line); + if (launchDeadline !== null) { + const remaining = launchDeadline - Date.now(); + if (remaining <= 0 || owner.signal.aborted) { + await emit({kind: "not_launched"} satisfies LaunchObservation); + return; + } + if (raw.timeout_ms !== null) raw.timeout_ms = Math.min(raw.timeout_ms, remaining); + } const lease = raw.delegated_lease; delete raw.delegated_lease; const request = decodeHostProcessRequest(raw); @@ -37,3 +56,4 @@ process.stdin.on("data", (chunk: Buffer) => { finally { process.stdin.destroy(); } })(); }); +if (launchDeadline !== null) await emit({kind: "ready"} satisfies LaunchObservation); diff --git a/loopx/control_plane/turn_driver/host_process_transport.py b/loopx/control_plane/turn_driver/host_process_transport.py index 4a169e0a12..f5b91f9677 100644 --- a/loopx/control_plane/turn_driver/host_process_transport.py +++ b/loopx/control_plane/turn_driver/host_process_transport.py @@ -7,6 +7,7 @@ import subprocess import sys import tempfile +import time from collections.abc import Callable, Sequence from contextlib import AbstractContextManager, ExitStack from pathlib import Path @@ -14,6 +15,7 @@ from ...file_lock import lock_holder_host_label from ..effect_runtime import _node_executable +from .host_bridge_transport import GuardedHostBridge # Keep Python's Windows executable/batch launcher compatibility. No timeout, # buffering or lifecycle decision lives in this transport-only child. @@ -212,6 +214,30 @@ def finish(self) -> None: self.pending = "" +def _begin_host_process_record(record_path: Path | None, *, record_value: str, + supervises: str, supervision: str, bridge_pid: int) -> dict[str, Any] | None: + if record_path is None: + return None + # Written before the request, so no Host exists that the record + # does not name; a record that cannot be written launches nothing. + record = {"schema_version": HOST_PROCESS_RECORD_SCHEMA_VERSION, + "host": lock_holder_host_label(), "owner_pid": os.getpid(), + "supervises": supervises, "supervision": supervision, "bridge_pid": bridge_pid, + "phase": "launching", "process_group": None} + if supervises == HOST_PROCESS_SUPERVISES_NESTED_HOST: + # Positive evidence for a CLI that exits before starting its + # Host. Missing evidence must never mean "not launched". + # Both writes precede the request that can start the CLI. + _write_host_process_record(Path(record_value), { + "schema_version": HOST_PROCESS_RECORD_SCHEMA_VERSION, + "host": lock_holder_host_label(), "owner_pid": os.getpid(), + "supervises": HOST_PROCESS_SUPERVISES_HOST, "supervision": "leased", + "phase": HOST_PROCESS_NOT_LAUNCHED, "process_group": None, + }) + _write_host_process_record(record_path, record) + return record + + def run_host_process( argv: Sequence[str], *, @@ -236,7 +262,7 @@ def run_host_process( command = list(argv) if sys.platform == "win32": command = [sys.executable, "-c", _WINDOWS_COMMAND_RELAY, *command] - request = { + request: dict[str, Any] = { "argv": command, "cwd": str(project), "input": input_text, @@ -266,12 +292,17 @@ def run_host_process( record_path = host_process_supervisor_record(record_path) supervises = HOST_PROCESS_SUPERVISES_NESTED_HOST supervision = "leased" - with subprocess.Popen( + launch_budget = max(1.0, timeout_seconds) if timeout_seconds is not None else 30.0 + launch_deadline = time.monotonic() + launch_budget if launch_guard is not None else None + launch_arguments = (["--guarded-launch", str((time.time() + launch_budget) * 1000)] + if launch_guard is not None else []) + proc = subprocess.Popen( [ _node_executable(), "--no-warnings", "--experimental-strip-types", str(bridge), + *launch_arguments, ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -281,78 +312,91 @@ def run_host_process( errors="strict", start_new_session=True, env=bridge_environment, - ) as proc: - assert proc.stdin is not None and proc.stdout is not None - result = None - record = None - launch_scope = ExitStack() - try: - if launch_guard is not None: - launch_scope.enter_context(launch_guard()) - if record_path is not None: - # Written before the request, so no Host exists that the record - # does not name; a record that cannot be written launches nothing. - record = {"schema_version": HOST_PROCESS_RECORD_SCHEMA_VERSION, - "host": lock_holder_host_label(), "owner_pid": os.getpid(), - "supervises": supervises, "supervision": supervision, "bridge_pid": proc.pid, - "phase": "launching", "process_group": None} - if supervises == HOST_PROCESS_SUPERVISES_NESTED_HOST: - # Positive evidence for a CLI that exits before starting its - # Host. Missing evidence must never mean "not launched". - # Both writes precede the request that can start the CLI. - _write_host_process_record(Path(record_value), { - "schema_version": HOST_PROCESS_RECORD_SCHEMA_VERSION, - "host": lock_holder_host_label(), "owner_pid": os.getpid(), - "supervises": HOST_PROCESS_SUPERVISES_HOST, "supervision": "leased", - "phase": HOST_PROCESS_NOT_LAUNCHED, "process_group": None, - }) - _write_host_process_record(record_path, record) - proc.stdin.write( - json.dumps(request, ensure_ascii=False, separators=(",", ":")) + "\n" - ) + ) + assert proc.stdin is not None and proc.stdout is not None + result: dict[str, Any] | None = None + record = None + launch_scope = ExitStack() + guarded = (GuardedHostBridge(proc, deadline=launch_deadline, + cleanup_timeout=8 if delegated_lease is not None else 5) + if launch_deadline is not None else None) + try: + if launch_guard is not None: + launch_scope.enter_context(launch_guard()) + record = _begin_host_process_record(record_path, record_value=record_value, + supervises=supervises, supervision=supervision, bridge_pid=proc.pid) + frame = json.dumps(request, ensure_ascii=False, separators=(",", ":")) + "\n" + if guarded is not None: + guarded.send(frame) + else: + proc.stdin.write(frame) proc.stdin.flush() - for line in proc.stdout: - event = json.loads(line) - kind = event.get("kind") - if kind == "spawned" and isinstance(event.get("pid"), int): - if record is not None: - group = event.get("process_group") - record.update(phase="spawned", host_pid=event["pid"], - process_group=group if isinstance(group, int) else None) - _write_host_process_record(record_path, record) - launch_scope.close() - elif kind in {"stdout", "stderr"} and isinstance(event.get("text"), str): - consume = on_stdout if kind == "stdout" else on_stderr - if consume is not None: - consume(event["text"]) - elif kind == "result" and event.get("outcome") in { - "exited", - "timeout", - "cancelled", - "output_limit", - "spawn_failed", - }: - result = event - launch_scope.close() - else: - raise RuntimeError("Invalid managed Host process observation") + lines = iter(proc.stdout) + while True: + event = (guarded.next(deadline=launch_deadline) if guarded is not None + else json.loads(line) if (line := next(lines, None)) is not None else None) + if event is None: + break + kind = event.get("kind") + if kind == "spawned" and isinstance(event.get("pid"), int): + if record is not None: + assert record_path is not None + group = event.get("process_group") + record.update(phase="spawned", host_pid=event["pid"], + process_group=group if isinstance(group, int) else None) + _write_host_process_record(record_path, record) + launch_deadline = None + launch_scope.close() + elif kind in {"stdout", "stderr"} and isinstance(event.get("text"), str): + consume = on_stdout if kind == "stdout" else on_stderr + if consume is not None: + consume(event["text"]) + elif kind == "result" and event.get("outcome") in { + "exited", + "timeout", + "cancelled", + "output_limit", + "spawn_failed", + }: + result = event + launch_deadline = None + launch_scope.close() + elif kind == "not_launched" and guarded is not None: + guarded.not_launched = True + raise TimeoutError("managed Host bridge launch deadline expired") + else: + raise RuntimeError("Invalid managed Host process observation") + finally: + try: + if guarded is not None and (launch_deadline is not None or sys.exc_info()[0] is not None): + # A failed handoff still owns the Goal fence. The TS bridge must + # report no launch or finish its Host cleanup before it releases. + guarded.cancel() + if record is not None and guarded.not_launched: + assert record_path is not None + record.update(phase=HOST_PROCESS_NOT_LAUNCHED, process_group=None) + record.pop("bridge_pid", None) + _write_host_process_record(record_path, record) + else: + # EOF asks the TS supervisor to clean its Host, also on a + # callback failure. Leased cleanup retains its original budget. + proc.stdin.close() + try: + proc.wait(timeout=8 if delegated_lease is not None else 5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() finally: launch_scope.close() - # Also runs on callback failure / Ctrl-C. Do not kill the supervisor - # before it has had a chance to terminate its owned Host group. - proc.stdin.close() - try: - # The leased CLI gets six seconds to acknowledge nested Host - # cleanup. Keep this control transport alive through its forced - # group kill as well, including callback failure / Ctrl-C. - proc.wait(timeout=8 if delegated_lease is not None else 5) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - if record is not None and result is not None and proc.returncode == 0: - # The supervisor returned only after cleaning its group; the group is still re-read. - record["phase"] = "finished" - _write_host_process_record(record_path, record) - if proc.returncode != 0 or result is None: - raise RuntimeError("Managed Host process supervision returned no result") - return result + if guarded is not None: + guarded.close_output() + else: + proc.stdout.close() + if record is not None and result is not None and proc.returncode == 0: + assert record_path is not None + # The supervisor returned only after cleaning its group; the group is still re-read. + record["phase"] = "finished" + _write_host_process_record(record_path, record) + if proc.returncode != 0 or result is None: + raise RuntimeError("Managed Host process supervision returned no result") + return result diff --git a/tests/test_host_launch_authority.py b/tests/test_host_launch_authority.py index 0571e78dd0..feaead81a4 100644 --- a/tests/test_host_launch_authority.py +++ b/tests/test_host_launch_authority.py @@ -3,10 +3,14 @@ from __future__ import annotations import contextlib +import hashlib import io import json +import os +import signal import sys import threading +import time from pathlib import Path import pytest @@ -20,6 +24,8 @@ source_session_registry_transaction, ) from loopx.control_plane.turn_driver import codex_cli, codex_operation_host +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission +from loopx.control_plane.turn_driver import host_process_transport from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime from tests.control_plane.test_effect_turn_live_quota_decision import ( _material_review_urgency, @@ -34,6 +40,129 @@ AGENT = "codex-fixture" +@pytest.mark.skipif(os.name == "nt", reason="Controlled POSIX bridge suspension") +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("boundary", ["startup", "body", "spawn"]) +def test_plain_host_bridge_backpressure_cannot_strand_the_stop_writer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, boundary: str, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, source = _write_live_fixture(tmp_path, provider=provider) + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=source, goal_id=GOAL, planned_goal_ref=None) + entered, stopping = threading.Event(), threading.Event() + bridge, completed, stopped = [], {}, {} + launched = tmp_path / "host-started" + real_popen = host_process_transport.subprocess.Popen + + def pause_bridge(*args, **kwargs): + process = real_popen(*args, **kwargs) + if any(str(arg).endswith("host_process_bridge.ts") for arg in args[0]): + # Pause only this test's actual Node bridge before it can consume + # the request. Its process group and pipes remain real. + if boundary == "startup": + os.kill(process.pid, signal.SIGSTOP) + bridge.append(process) + return process + + monkeypatch.setattr(host_process_transport.subprocess, "Popen", pause_bridge) + environment = dict(os.environ) + if boundary != "startup": + preload = tmp_path / "pause-bridge.mjs" + preload.write_text('process.stdin.once("data", () => process.kill(process.pid, "SIGSTOP"));') + environment["NODE_OPTIONS"] = f"--import={preload}" + + @contextlib.contextmanager + def guard(): + with admission.launch_lifetime(operation="synthetic_bridge_backpressure"): + entered.set() + yield + + def run(): + try: + completed["result"] = host_process_transport.run_host_process( + [sys.executable, "-c", f"from pathlib import Path;Path({str(launched)!r}).touch()"], + project=project, input_text="界" * (128 if boundary == "spawn" else 1024 * 1024), + timeout_seconds=1, launch_guard=guard, environment=environment) + except Exception as error: + completed["error"] = error + + def stop(): + stopping.set() + try: + stopped["result"] = activation_service.set_goal_activation_state( + registry_path=source, runtime_root_override=str(runtime), goal_id=GOAL, + state="stopped", reason="Synthetic Stop during bridge dispatch.", + actor_kind="owner", execute=True) + except Exception as error: + stopped["error"] = error + + runner = threading.Thread(target=run) + stopper = threading.Thread(target=stop) + runner.start() + try: + assert entered.wait(10) + stopper.start() + assert stopping.wait(1) + runner.join(timeout=4) + stopper.join(timeout=1) + bounded = not runner.is_alive() and not stopper.is_alive() + finally: + # A pre-fix failure must not leave a suspended owned process or block + # the test session. This deliberate recovery is never success evidence. + for process in bridge: + if process.poll() is None: + os.kill(process.pid, signal.SIGCONT) + runner.join(timeout=15) + if stopper.ident is not None: + stopper.join(timeout=15) + assert bounded, "bridge request delivery stranded the Goal Stop writer" + assert isinstance(completed.get("error"), TimeoutError), completed + assert stopped["result"]["readback"]["verified"] is True, stopped + assert all(process.poll() is not None for process in bridge) + assert not launched.exists(), "a timed-out bridge launched a late Host" + + +def test_plain_host_bridge_preserves_slow_healthy_unicode_delivery(tmp_path, monkeypatch): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, _runtime, source = _write_live_fixture(tmp_path, provider="file") + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=source, goal_id=GOAL, planned_goal_ref=None) + preload = tmp_path / "slow-bridge.mjs" + preload.write_text('process.stdin.once("data", () => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1200));') + text = "界🙂\n" * 70000 + output = [] + started = time.monotonic() + observed = host_process_transport.run_host_process( + [sys.executable, "-c", "import hashlib,sys;print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest())"], + project=project, input_text=text, timeout_seconds=10, + on_stdout=output.append, environment=dict(os.environ, NODE_OPTIONS=f"--import={preload}"), + launch_guard=lambda: admission.launch_lifetime(operation="synthetic_slow_bridge")) + assert time.monotonic() - started >= 1.2 + assert observed["outcome"] == "exited" and observed["returncode"] == 0, observed + assert "".join(output).strip() == hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@pytest.mark.skipif(os.name == "nt", reason="Controlled POSIX bridge cancellation") +def test_plain_host_bridge_does_not_certify_unconfirmed_cleanup(tmp_path, monkeypatch): + from loopx.control_plane.turn_driver.host_process_transport import execution_host_drain + + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, source = _write_live_fixture(tmp_path, provider="file") + admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=source, goal_id=GOAL, planned_goal_ref=None) + preload = tmp_path / "unresponsive-bridge.mjs" + preload.write_text('process.stdin.once("data", () => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 30000));') + record = runtime / "unconfirmed-host.json" + with pytest.raises(RuntimeError, match="cleanup unconfirmed"): + host_process_transport.run_host_process( + [sys.executable, "-c", "pass"], project=project, input_text="界" * 1024 * 1024, + timeout_seconds=1, environment=dict(os.environ, NODE_OPTIONS=f"--import={preload}", + LOOPX_HOST_PROCESS_RECORD=str(record)), + launch_guard=lambda: admission.launch_lifetime(operation="synthetic_unresponsive_bridge")) + assert execution_host_drain(record) == "unattributable" + + def _cli(registry: Path, runtime: Path, *arguments: str): output = io.StringIO() with contextlib.redirect_stdout(output): From 4115350557abc626028002c6cb6bd0f98ad573f8 Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 19:38:08 +0800 Subject: [PATCH 5/6] fix(host): retire the native peer when admission exhausts its deadline Signed-off-by: song --- docs/integrations/runtime-connector-catalog.md | 3 +++ loopx/chat_agent.py | 9 +++++++++ tests/control_plane/test_codex_operation_launch.py | 11 +++++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/integrations/runtime-connector-catalog.md b/docs/integrations/runtime-connector-catalog.md index d726eb3fa0..147782705c 100644 --- a/docs/integrations/runtime-connector-catalog.md +++ b/docs/integrations/runtime-connector-catalog.md @@ -107,6 +107,8 @@ native peer and drains the writer before releasing the fence; it cannot leave a sender to deliver that Turn after Stop. This is a dispatch timeout, not proof of non-acceptance or permission to replay. Successful large and Unicode inputs retain the full original text stream; no new input-size truncation is applied. +If current admission consumes the remaining request deadline before a writer +starts, the Session owner still retires that same peer inside the launch fence. The guarded ordinary-Host bridge also bounds readiness, request delivery and spawn observation within the original explicit command deadline. Without a @@ -139,6 +141,7 @@ Codex operation tools 路径在创建 app-server 和发送 `turn/start` 前复 原生写入也计入该界限,不再对正常发送额外施加一秒上限。背压或部分发送中断时,先取消并回收同一个 原生进程、等待写入线程退出,再释放 Goal 锁,避免 Stop 后迟到派发。这类超时不 证明请求从未受理,也不授权重放;正常的大输入和 Unicode 文本完整保留,不截断。 +当前资格检查在写入开始前耗尽原请求期限时,会话 owner 仍在启动锁内回收同一个进程。 普通 Host 的受控 bridge 也在原显式命令期限内完成 ready、请求发送和 spawn 观察。 没有命令期限时,仅交接阶段有 30 秒安全预算;成功启动后仍等待 Host 自然结束。 diff --git a/loopx/chat_agent.py b/loopx/chat_agent.py index 227d3724b2..ff5d45109f 100644 --- a/loopx/chat_agent.py +++ b/loopx/chat_agent.py @@ -1134,6 +1134,15 @@ def _guarded_write( with write_guard(): process = self.process try: + if time.monotonic() >= dispatch_deadline: + # Canonical admission itself can consume the remaining + # budget. No writer started, but this Session still + # retires its peer before releasing the launch fence. + try: + force_terminate_process(process, process_tree_owned=self.process_tree_owned) + except Exception as error: + raise RuntimeError("native input receiver cleanup unconfirmed") from error + raise TimeoutError("native input dispatch deadline expired") write_process_stdin_before_deadline( process, frame, deadline=dispatch_deadline, cancel_and_reap=lambda: force_terminate_process( diff --git a/tests/control_plane/test_codex_operation_launch.py b/tests/control_plane/test_codex_operation_launch.py index d0388e61e6..cb2016de9e 100644 --- a/tests/control_plane/test_codex_operation_launch.py +++ b/tests/control_plane/test_codex_operation_launch.py @@ -230,8 +230,9 @@ def guard(): @pytest.mark.parametrize("provider", ["file", "sqlite"]) -def test_backpressured_native_turn_drains_before_stop_writer_is_released( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, +@pytest.mark.parametrize("dispatch_window", ["write_backpressure", "admission_deadline"]) +def test_native_turn_dispatch_failure_drains_before_stop_writer_is_released( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str, dispatch_window: str, ) -> None: """A full stdin pipe cannot hold Stop or deliver a late native Turn.""" runtime, registry, _, turns, arguments = _fixture(tmp_path, monkeypatch, provider) @@ -266,6 +267,12 @@ def lifetime(self, *, operation): return entered.set() try: + if dispatch_window == "admission_deadline": + # Expiry can occur in canonical admission before the first + # write. The Session owner still retires its original peer + # before releasing the fence; no helper-side partial write + # is needed to expose this boundary. + time.sleep(1.1) yield finally: drained_before_release.append(sessions[0].process.poll() is not None) From 1be62b700290f8c263ad2b68d3ac617487313d54 Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 19:50:13 +0800 Subject: [PATCH 6/6] test(host): recognize the merged execution observation digest consumer Signed-off-by: song --- tests/control_plane_ts/content_digest_single_owner.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index 8a3827f6a7..990b365ff8 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -93,6 +93,7 @@ const CANONICAL_CONSUMERS = [ "control_plane/agents/supervisor_event_append.ts", "control_plane/capabilities/change_quality_validation.ts", "control_plane/capabilities/content_media.ts", + "control_plane/capabilities/explore_execution_observation.ts", "control_plane/capabilities/explore_result_page.ts", "control_plane/capabilities/external_evidence.ts", "control_plane/capabilities/goal_capability_organization.ts",