diff --git a/benchmark/runtime/sforge.py b/benchmark/runtime/sforge.py index 1218ba3a3c..a9b983f4f9 100644 --- a/benchmark/runtime/sforge.py +++ b/benchmark/runtime/sforge.py @@ -216,6 +216,10 @@ def __init__(self, config, *, profile: str, cwd: str, # workers already run a scheduler: restarting it after terminal/quiescent # exit duplicates that owner and can exhaust the native resume limit. self.resume_cmd = CodexAgent.resume_cmd if profile == "official" else None + # SForge signs this declaration into the runtime protocol. The setup + # callback below is shared by every profile, but only the official + # profile installs the Codex stop hook inside that callback. + self.stop_hook = CodexAgent.stop_hook if profile == "official" else None def install_stop_hook(self, backend, handle, log_dir, logger): self.environment = SForgeEnvironment( diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index 60dae42b34..e8f1ca0fe5 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -77,7 +77,8 @@ def test_scheduler_modes_do_not_add_an_outer_loop_to_native_goal(): ("official", True), ("single", False), ("native-goal", False), ("heartbeat-resume", False), ("heartbeat-explore", False), ]) -def test_only_official_delegates_continuation_to_sforge(profile, resumes, monkeypatch): +def test_only_official_delegates_continuation_and_stop_hook_to_sforge( + profile, resumes, monkeypatch): pytest.importorskip("sforge") pytest.importorskip("harbor") from sforge.harness.config import SForgeConfig @@ -86,6 +87,7 @@ def test_only_official_delegates_continuation_to_sforge(profile, resumes, monkey worker = SForgeWorker(SForgeConfig(agent_model="fixture-model", agent_effort="xhigh"), profile=profile, cwd="/task") assert (worker.resume_cmd is not None) is resumes + assert (worker.stop_hook is not None) is (profile == "official") assert worker.timeout_seconds == 64800 assert worker.task_entry == ("loopx-planned" if profile.startswith("heartbeat-") else "seeded-todo") if profile in {"official", "single"}: @@ -94,6 +96,32 @@ def test_only_official_delegates_continuation_to_sforge(profile, resumes, monkey assert 'web_search="disabled"' in command +@pytest.mark.parametrize("profile", [ + "official", "single", "native-goal", "heartbeat-resume", "heartbeat-explore", +]) +def test_runtime_protocol_attests_only_official_stop_hook(profile, monkeypatch): + from types import SimpleNamespace + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from sforge.harness.config import SForgeConfig + run_agent = pytest.importorskip("sforge.harness.run_agent") + runtime_protocol = getattr(run_agent, "_runtime_protocol", None) + if runtime_protocol is None: + pytest.skip("SForge provider does not expose the signed runtime protocol builder") + from benchmark.runtime.sforge import SForgeWorker + monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/private-credential") + worker = SForgeWorker(SForgeConfig(agent_model="fixture-model", agent_effort="xhigh"), + profile=profile, cwd="/task") + protocol = runtime_protocol( + agent=worker, backend=SimpleNamespace(backend_name="docker"), + model="fixture-model", effective_timeout=64800, eval_interval=300, + disable_stop_hook=False, disable_auto_eval=False, + disable_auto_resume=False, internet=False, max_submissions=None, + submission_cooldown=3600, + ) + assert protocol["stop_hook_enabled"] is (profile == "official") + + def test_invalid_worker_inputs_fail_before_install(monkeypatch): pytest.importorskip("sforge") pytest.importorskip("harbor") @@ -353,7 +381,7 @@ async def installed(self, environment): assert profile.get("native_goal_continue") is (True if enabled else None) with pytest.raises(ValueError, match="native-goal"): SForgeWorker(config, profile="heartbeat-resume", cwd="/task", native_goal_continue=True) - with pytest.raises(ValueError, match="context_delivery"): + with pytest.raises(ValueError, match="native_goal_continue"): SForgeWorker(config, profile="native-goal", cwd="/task", native_goal_continue="1")