From 671e10a3f457c5bb335855e6a89cab8d74e6a2fe Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sun, 25 Oct 2026 14:35:50 +0800 Subject: [PATCH 1/2] fix(benchmark): attest the effective stop hook profile Signed-off-by: huangruiteng --- benchmark/runtime/sforge.py | 4 ++++ benchmark/tests/test_sforge_runtime.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/benchmark/runtime/sforge.py b/benchmark/runtime/sforge.py index 1218ba3a3c..a9b983f4f9 100644 --- a/benchmark/runtime/sforge.py +++ b/benchmark/runtime/sforge.py @@ -216,6 +216,10 @@ def __init__(self, config, *, profile: str, cwd: str, # workers already run a scheduler: restarting it after terminal/quiescent # exit duplicates that owner and can exhaust the native resume limit. self.resume_cmd = CodexAgent.resume_cmd if profile == "official" else None + # SForge signs this declaration into the runtime protocol. The setup + # callback below is shared by every profile, but only the official + # profile installs the Codex stop hook inside that callback. + self.stop_hook = CodexAgent.stop_hook if profile == "official" else None def install_stop_hook(self, backend, handle, log_dir, logger): self.environment = SForgeEnvironment( diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index 60dae42b34..33526188a6 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -77,15 +77,26 @@ def test_scheduler_modes_do_not_add_an_outer_loop_to_native_goal(): ("official", True), ("single", False), ("native-goal", False), ("heartbeat-resume", False), ("heartbeat-explore", False), ]) -def test_only_official_delegates_continuation_to_sforge(profile, resumes, monkeypatch): +def test_only_official_delegates_continuation_and_stop_hook_to_sforge( + profile, resumes, monkeypatch): + from types import SimpleNamespace pytest.importorskip("sforge") pytest.importorskip("harbor") from sforge.harness.config import SForgeConfig + from sforge.harness.run_agent import _runtime_protocol from benchmark.runtime.sforge import SForgeWorker monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/private-credential") worker = SForgeWorker(SForgeConfig(agent_model="fixture-model", agent_effort="xhigh"), profile=profile, cwd="/task") assert (worker.resume_cmd is not None) is resumes + protocol = _runtime_protocol( + agent=worker, backend=SimpleNamespace(backend_name="docker"), + model="fixture-model", effective_timeout=64800, eval_interval=300, + disable_stop_hook=False, disable_auto_eval=False, + disable_auto_resume=False, internet=False, max_submissions=None, + submission_cooldown=3600, + ) + assert protocol["stop_hook_enabled"] is (profile == "official") assert worker.timeout_seconds == 64800 assert worker.task_entry == ("loopx-planned" if profile.startswith("heartbeat-") else "seeded-todo") if profile in {"official", "single"}: @@ -353,7 +364,7 @@ async def installed(self, environment): assert profile.get("native_goal_continue") is (True if enabled else None) with pytest.raises(ValueError, match="native-goal"): SForgeWorker(config, profile="heartbeat-resume", cwd="/task", native_goal_continue=True) - with pytest.raises(ValueError, match="context_delivery"): + with pytest.raises(ValueError, match="native_goal_continue"): SForgeWorker(config, profile="native-goal", cwd="/task", native_goal_continue="1") From 324c53c7bbfeadb62019d3d1911c36136caa14f0 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Mon, 26 Oct 2026 16:15:12 +0800 Subject: [PATCH 2/2] test(benchmark): retain compatible SForge profile coverage Signed-off-by: huangruiteng --- benchmark/tests/test_sforge_runtime.py | 35 +++++++++++++++++++------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index 33526188a6..e8f1ca0fe5 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -79,17 +79,40 @@ def test_scheduler_modes_do_not_add_an_outer_loop_to_native_goal(): ]) def test_only_official_delegates_continuation_and_stop_hook_to_sforge( profile, resumes, monkeypatch): - from types import SimpleNamespace pytest.importorskip("sforge") pytest.importorskip("harbor") from sforge.harness.config import SForgeConfig - from sforge.harness.run_agent import _runtime_protocol from benchmark.runtime.sforge import SForgeWorker monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/private-credential") worker = SForgeWorker(SForgeConfig(agent_model="fixture-model", agent_effort="xhigh"), profile=profile, cwd="/task") assert (worker.resume_cmd is not None) is resumes - protocol = _runtime_protocol( + assert (worker.stop_hook is not None) is (profile == "official") + assert worker.timeout_seconds == 64800 + assert worker.task_entry == ("loopx-planned" if profile.startswith("heartbeat-") else "seeded-todo") + if profile in {"official", "single"}: + command = worker.format_run_cmd("/task.md", internet=False) + assert 'model_reasoning_effort="xhigh"' in command + assert 'web_search="disabled"' in command + + +@pytest.mark.parametrize("profile", [ + "official", "single", "native-goal", "heartbeat-resume", "heartbeat-explore", +]) +def test_runtime_protocol_attests_only_official_stop_hook(profile, monkeypatch): + from types import SimpleNamespace + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from sforge.harness.config import SForgeConfig + run_agent = pytest.importorskip("sforge.harness.run_agent") + runtime_protocol = getattr(run_agent, "_runtime_protocol", None) + if runtime_protocol is None: + pytest.skip("SForge provider does not expose the signed runtime protocol builder") + from benchmark.runtime.sforge import SForgeWorker + monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/private-credential") + worker = SForgeWorker(SForgeConfig(agent_model="fixture-model", agent_effort="xhigh"), + profile=profile, cwd="/task") + protocol = runtime_protocol( agent=worker, backend=SimpleNamespace(backend_name="docker"), model="fixture-model", effective_timeout=64800, eval_interval=300, disable_stop_hook=False, disable_auto_eval=False, @@ -97,12 +120,6 @@ def test_only_official_delegates_continuation_and_stop_hook_to_sforge( submission_cooldown=3600, ) assert protocol["stop_hook_enabled"] is (profile == "official") - assert worker.timeout_seconds == 64800 - assert worker.task_entry == ("loopx-planned" if profile.startswith("heartbeat-") else "seeded-todo") - if profile in {"official", "single"}: - command = worker.format_run_cmd("/task.md", internet=False) - assert 'model_reasoning_effort="xhigh"' in command - assert 'web_search="disabled"' in command def test_invalid_worker_inputs_fail_before_install(monkeypatch):