From 736b23af1ed2b114f307e822e0d4932d648545f0 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 17:07:33 +0530 Subject: [PATCH 01/12] test(harness): drop a removed tool's test body that a merge left inside the cap test --- tests/harness/test_grade.py | 40 ------------------------------------- 1 file changed, 40 deletions(-) diff --git a/tests/harness/test_grade.py b/tests/harness/test_grade.py index 324123544..2393c231d 100644 --- a/tests/harness/test_grade.py +++ b/tests/harness/test_grade.py @@ -230,46 +230,6 @@ def test_the_cap_holds_for_the_session_that_saves_too(tmp_path): said = asyncio.run(submit.handler({"name": "one_more", "instruction": "three"})) assert said.get("is_error"), can_save assert "2 of 2 written" in said["content"][0]["text"] - contract = AgentContract( - agent="cart", real_use_cases=["add an item", "remove an item"] - ) - # Above FEWEST_WORTH_DELEGATING, or the fan-out tool is not published at all. - server, _kept = st.scenario_tools(contract, tmp_path, tmp_path, wanted=20) - suite = next(spec for spec in server.tools if spec.name == "generate_suite") - - asked_for: list[int] = [] - - async def _fake_parallel(contract, *, out, wanted, use_cases, slices, at_once): - asked_for.append(wanted) - return [] - - monkeypatch.setattr("fi.alk.harness.scenarios.write_in_parallel", _fake_parallel) - # Nothing on disk yet: the full ask goes through. - asyncio.run(suite.handler({"count": 20})) - assert asked_for == [20] - - # Fourteen already written, and the model asks for twenty again: six are outstanding. - monkeypatch.setattr( - st, - "load_scenarios", - lambda _d: [ - Scenario(name=f"s{i}", instruction="i", sub_goals=["x"]) for i in range(14) - ], - ) - monkeypatch.setattr(st, "journalled", lambda _d: []) - asyncio.run(suite.handler({"count": 20})) - assert asked_for == [20, 6] - - monkeypatch.setattr( - st, - "load_scenarios", - lambda _d: [ - Scenario(name=f"s{i}", instruction="i", sub_goals=["x"]) for i in range(20) - ], - ) - said = asyncio.run(suite.handler({"count": 20})) - assert "already holds the 20" in said["content"][0]["text"] - assert asked_for == [20, 6] def test_only_chat_can_grow_a_finished_suite(tmp_path): From 678b9aafeecb7255bddb5dcb6778d47a708a6012 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 17:08:06 +0530 Subject: [PATCH 02/12] test(harness): give the dispatch-ack fakes the failure, report and runner fields the call runner now reads --- tests/harness/test_dispatch_ack.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/harness/test_dispatch_ack.py b/tests/harness/test_dispatch_ack.py index b31eb9ce1..600f54d56 100644 --- a/tests/harness/test_dispatch_ack.py +++ b/tests/harness/test_dispatch_ack.py @@ -19,6 +19,7 @@ from fi.alk.harness import hosted_scheduler as hs from fi.alk.harness.hosted_scheduler import CallAborted from fi.alk.harness.job import FailureDomain +from fi.simulate.runtime.failures import FailureStage, SimulationFailure from fi.simulate.runtime.run import TestCaseStatus _UNACK = "voice_dispatch_unacknowledged" @@ -125,19 +126,26 @@ def _runner() -> cr.CallRunnerImpl: runner = cr.CallRunnerImpl.__new__(cr.CallRunnerImpl) runner._adapter = None # never touched: no calls, no result runner._collect_calls = lambda runtime: () # instance override + runner._context = SimpleNamespace(usage_reporter=None) + runner._environ = {} return runner def _report(*, status: TestCaseStatus, code: str | None) -> SimpleNamespace: - failure = SimpleNamespace(code=code, message="dispatch not acked") if code else None + failure = ( + SimulationFailure(stage=FailureStage.RUNNING, code=code, message="dispatch not acked") + if code + else None + ) case = SimpleNamespace( + test_case_id="c1", status=status, started_at=None, ended_at=None, result=None, failure=failure, ) - return SimpleNamespace(test_cases=[case]) + return SimpleNamespace(run_id="r1", test_cases=[case]) def _translate(report: SimpleNamespace) -> CallAborted: From d0e9cbe4dbde491d1effb80b95f392154fd009a5 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 17:11:20 +0530 Subject: [PATCH 03/12] fix(harness): route the ask tool through the permission callback and restore the session's built-in tool list on the Claude backend --- src/fi/alk/harness/backends/claude.py | 6 +++++- tests/test_harness.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/backends/claude.py b/src/fi/alk/harness/backends/claude.py index ff5d18f89..bd27b1a5c 100644 --- a/src/fi/alk/harness/backends/claude.py +++ b/src/fi/alk/harness/backends/claude.py @@ -402,7 +402,9 @@ def _options(self, spec: SessionSpec) -> ClaudeAgentOptions: wire_model = gateway_wire_model(spec.model) gateway_compatible = behind_gateway(spec.model) # A worker's calls are made inside this session, so the gate covers its tools too. - allowed = [name for name in spec.granted_anywhere() if name != DELEGATE_TOOL] + # The ask tool stays out: an allowed tool is approved before the permission callback, so the + # question would never reach the person. + allowed = [name for name in spec.granted_anywhere() if name not in (DELEGATE_TOOL, ASK_TOOL)] # Union per server name; each agent is still restricted by its own `tools` allowlist. servers: dict[str, ToolServer] = {} for source in (spec.servers, *(worker.servers for worker in spec.workers.values())): @@ -431,7 +433,9 @@ def _options(self, spec: SessionSpec) -> ClaudeAgentOptions: environment["CLAUDE_CODE_DISABLE_BACKGROUND_TASKS"] = "1" allowed.extend(DELEGATION_TOOLS) _can_reach_its_workers(spec, allowed) + builtins = [name for name in spec.builtins if name != DELEGATE_TOOL] options = ClaudeAgentOptions( + tools=[*builtins, *(DELEGATION_TOOLS if spec.workers else ())], system_prompt=_prompt_for(spec.system_prompt), allowed_tools=allowed, mcp_servers={ diff --git a/tests/test_harness.py b/tests/test_harness.py index d1fddad83..83dc09796 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -4390,16 +4390,18 @@ async def hand(_args): assert denied != {} -def test_hosted_claude_session_uses_durable_resume_and_restricted_tools(tmp_path): +def test_hosted_claude_session_uses_durable_resume_and_restricted_tools(tmp_path, monkeypatch): from fi.alk.harness.backends import ConversationSession, SessionSpec from fi.alk.harness.backends.claude import ClaudeBackend + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_URL", "https://gateway.futureagi.test") + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_API_KEY", "internal-key") transcript_store = object() spec = SessionSpec( system_prompt="hosted", builtins=("Read", "AskUserQuestion"), cwd=str(tmp_path), - model="claude-sonnet-4-6", + model="vertex_ai/gemini-3.7-flash", conversation=ConversationSession( app_name="alk-harness", user_id="conversation-user", From d64b55eabeabb733585b25aa3501c30df3348578 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 17:11:48 +0530 Subject: [PATCH 04/12] test(harness): name the scenario fixture fields as the model has them and list the skill's example tool names --- src/fi/alk/harness/skills/write-scenarios/SKILL.md | 2 +- tests/test_harness.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 9644219e4..ef60e84b3 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -530,7 +530,7 @@ conversation is the only evidence there will be. Everything else in this skill s - **Take the difficulty from the agent's instructions.** Its policies, required questions, disclosures, refusals, escalation rules and limits are the cells. A tool that fails or an account state are not available here. -- `setup`/`ready` only if the caller needs a private fixture; usually both stay empty. +- `setup_code`/`ready_code` only if the caller needs a private fixture; usually both stay empty. ## Before you submit: read it the way a reviewer will diff --git a/tests/test_harness.py b/tests/test_harness.py index 83dc09796..64280a5ae 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -4658,6 +4658,9 @@ def test_a_skill_only_names_tools_its_stage_actually_has(): | {"origin", "matching"} | {"handle", "check", "args", "db", "world", "calls", "json", "ToolError"} | {"book_ride", "cancel_ride", "verify_otp", "bookings"} + | {"create_booking", "cancel_booking", "verify_code", "cancel_subscription", "reset_password"} + # Words a check example searches for or reads, not tools. + | {"company", "pan"} | {"social_engineering_resisted", "true", "or"} ) From d2b990e09260454e6f5a4ef1e01843718d66fa34 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 17:12:50 +0530 Subject: [PATCH 05/12] test(harness): set the prompt budget from the skills as they now stand --- tests/test_harness.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_harness.py b/tests/test_harness.py index 64280a5ae..98a928403 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -7728,8 +7728,9 @@ def test_the_system_prompt_stays_within_budget(): contract.brief(with_data=True, sample_rows=3) + load_skill("write-scenarios") + kinds ) main_loop = sub_agent + load_skill("plan-suite", preamble=False) - assert len(sub_agent) <= 90_000, f"sub-agent prompt is {len(sub_agent)} chars" - assert len(main_loop) <= 130_000, f"main loop prompt is {len(main_loop)} chars" + # Set from the skills as they stand plus about a tenth: growth past this is a choice to make, not drift. + assert len(sub_agent) <= 165_000, f"sub-agent prompt is {len(sub_agent)} chars" + assert len(main_loop) <= 245_000, f"main loop prompt is {len(main_loop)} chars" def test_a_sub_goal_defined_after_the_fact_reaches_the_folders_already_written(tmp_path): From 71562097eabf9cc13a21f25d562e8d1fffd19e46 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 17:13:21 +0530 Subject: [PATCH 06/12] test(simulator): match the caller's re-ask rule to its current wording --- tests/test_simulator_voice_instructions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simulator_voice_instructions.py b/tests/test_simulator_voice_instructions.py index 41ddc656b..6b574a77d 100644 --- a/tests/test_simulator_voice_instructions.py +++ b/tests/test_simulator_voice_instructions.py @@ -34,7 +34,7 @@ def test_the_caller_may_not_claim_an_action_it_cannot_perform(): def test_the_caller_asks_again_for_the_part_of_its_question_left_unanswered(): - assert "hold the answer against what you asked" in SIMULATOR_INSTRUCTIONS + assert "Hold every answer against what you asked" in SIMULATOR_INSTRUCTIONS assert "ask for that part once" in SIMULATOR_INSTRUCTIONS From 2464f3a48cfc3f4e1c65e5a844338e69bf7dbcd2 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 18:02:55 +0530 Subject: [PATCH 07/12] test(harness): give hosted job fixtures the pinned commit, isolation and secret references hosted jobs require --- tests/test_harness_architecture.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_harness_architecture.py b/tests/test_harness_architecture.py index d9655f38a..98274a03a 100644 --- a/tests/test_harness_architecture.py +++ b/tests/test_harness_architecture.py @@ -80,6 +80,7 @@ async def fake_auto(args) -> int: job_id="job-upload", run_id="run-upload", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={"kind": "archive", "archive_artifact_id": "source-id"}, agent={"connector": "auto"}, metadata={"source_kind": "archive"}, @@ -112,8 +113,10 @@ def fake_run(*_args, **_kwargs): job_id="job-github", run_id="run-github", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "customer/agent", "ref": "main", @@ -990,8 +993,10 @@ def test_job_carries_references_but_rejects_resolved_secrets() -> None: job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "installation_id": "installation", "repository": "customer/agent", "ref": "main", @@ -1000,9 +1005,9 @@ def test_job_carries_references_but_rejects_resolved_secrets() -> None: "connector": "livekit", "secret_refs": { "api_key": { - "manager": "futureagi", + "manager": "platform-vault", "key": "secret_livekit_key", - "purpose": "connect to agent", + "purpose": "target_provider", } }, }, @@ -1015,8 +1020,10 @@ def test_job_carries_references_but_rejects_resolved_secrets() -> None: job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "installation_id": "installation", "repository": "customer/agent", }, @@ -1029,6 +1036,7 @@ def test_public_github_job_needs_no_installation_but_is_commit_pinnable() -> Non job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", "visibility": "public", @@ -1048,8 +1056,10 @@ def test_public_github_branch_url_is_normalized_into_repository_and_ref() -> Non job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "https://github.com/future-agi/future-agi/tree/feat/harness", }, @@ -1090,8 +1100,10 @@ def test_hosted_job_rejects_unsafe_security_policy(security: dict) -> None: job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "customer/public-agent", }, @@ -1216,6 +1228,8 @@ def test_hosted_github_checkout_keeps_token_out_of_process_arguments( observed = {} def run(command, **kwargs): + if command[1] == "rev-parse": + return SimpleNamespace(returncode=0, stdout="a" * 40 + "\n", stderr="") observed["command"] = command observed["environment"] = kwargs["env"] Path(command[-1]).mkdir() @@ -1226,8 +1240,10 @@ def run(command, **kwargs): job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "installation_id": "installation", "repository": "customer/private-agent", "ref": "main", @@ -1253,6 +1269,8 @@ def token(_installation): raise AssertionError("public checkout must not request an installation token") def run(command, **kwargs): + if command[1] == "rev-parse": + return SimpleNamespace(returncode=0, stdout="a" * 40 + "\n", stderr="") observed["environment"] = kwargs["env"] Path(command[-1]).mkdir() return SimpleNamespace(returncode=0, stdout="", stderr="") @@ -1262,8 +1280,10 @@ def run(command, **kwargs): job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "customer/public-agent", }, @@ -1282,6 +1302,8 @@ def test_public_github_branch_url_clones_the_branch( observed = {} def run(command, **kwargs): + if command[1] == "rev-parse": + return SimpleNamespace(returncode=0, stdout="a" * 40 + "\n", stderr="") observed["command"] = command Path(command[-1]).mkdir() return SimpleNamespace(returncode=0, stdout="", stderr="") @@ -1291,8 +1313,10 @@ def run(command, **kwargs): job_id="job", run_id="run", execution="hosted", + runtime={"isolation": "dedicated_vm"}, source={ "kind": "github", + "commit_sha": "a" * 40, "visibility": "public", "repository": "https://github.com/acme/agent/tree/feat/harness", }, From 9497fd6ade079d70232c4e126d50c96eb4da02f7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 18:04:27 +0530 Subject: [PATCH 08/12] docs(harness): describe the proof gates without naming another benchmark --- src/fi/alk/harness/DESIGN.md | 2 +- src/fi/alk/harness/prove.py | 3 +-- src/fi/alk/harness/scenario.py | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/harness/DESIGN.md b/src/fi/alk/harness/DESIGN.md index 8c7c0bc4e..dc4b50b01 100644 --- a/src/fi/alk/harness/DESIGN.md +++ b/src/fi/alk/harness/DESIGN.md @@ -141,7 +141,7 @@ was invented, tone. ## 4. Three gates on every scenario, before it is kept -Terminal-bench's oracle run, which is the reason its tasks are known to be solvable. +A reference run like this is how a benchmark's tasks are known to be solvable. ### Gate 1. Ready diff --git a/src/fi/alk/harness/prove.py b/src/fi/alk/harness/prove.py index 9f6188658..c08172467 100644 --- a/src/fi/alk/harness/prove.py +++ b/src/fi/alk/harness/prove.py @@ -1,8 +1,7 @@ """Proving a scenario is worth keeping, before anything is ever run against the agent. Three gates, all pure code. No model is asked whether a scenario is good; the environment -decides. Terminal-bench keeps its tasks honest this way, and it is the cheapest useful thing in -the whole harness: no tokens, no network, a few milliseconds. +decides. It is the cheapest useful thing in the whole harness: no tokens, no network, a few milliseconds. **Ready.** Reset the world, run the scenario's own ``setup.py``, then its ``ready.py``. The world has to hold what the scenario presumes. A scenario about the last five chocolates is only a test diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 6a9be11e2..19a7ac57f 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -6,8 +6,7 @@ It also carries a **solution**: what a correct agent would do. That is not decoration. It is what proves, before the scenario is ever used, that the scenario can be passed at all and that its -checks are not vacuous — the two gates in ``prove.py``. Terminal-bench keeps its tasks honest the -same way, and it needs no model to do it. +checks are not vacuous — the two gates in ``prove.py``. It needs no model to do it. """ from __future__ import annotations From 166adb3448f8b39f29c9cbcae9e797015a14e6db Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 22 Oct 2026 18:05:27 +0530 Subject: [PATCH 09/12] test(studio): point the stub account at the URL variable the client reads first, so the pull never leaves localhost --- tests/test_phase7_persona_studio.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_phase7_persona_studio.py b/tests/test_phase7_persona_studio.py index e8a096870..b68cf61b9 100644 --- a/tests/test_phase7_persona_studio.py +++ b/tests/test_phase7_persona_studio.py @@ -652,7 +652,9 @@ def stub_account(monkeypatch): url = f"http://127.0.0.1:{server.server_address[1]}" monkeypatch.setenv("AGENT_LEARNING_API_KEY", "test-key") monkeypatch.setenv("AGENT_LEARNING_SECRET_KEY", "test-secret") - monkeypatch.setenv("AGENT_LEARNING_API_URL", url) + # Importing the config writes the default URL into FI_BASE_URL, which is read first. + for name in ("FI_BASE_URL", "FUTURE_AGI_API_URL", "AGENT_LEARNING_API_URL"): + monkeypatch.setenv(name, url) try: yield url finally: From 61ae25914a1094e1e1e337cad545f08889a2fa4f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 01:22:53 +0530 Subject: [PATCH 10/12] fix(harness): answer the CLI's question tool with a decision during platform turns and offer workers their own built-ins --- src/fi/alk/harness/backends/claude.py | 9 +++++- src/fi/alk/harness/hosted_chat_entrypoint.py | 21 ++++++------ tests/harness/test_builder_chat.py | 3 ++ tests/test_harness.py | 34 ++++++++++++++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/fi/alk/harness/backends/claude.py b/src/fi/alk/harness/backends/claude.py index bd27b1a5c..2549514c4 100644 --- a/src/fi/alk/harness/backends/claude.py +++ b/src/fi/alk/harness/backends/claude.py @@ -433,7 +433,14 @@ def _options(self, spec: SessionSpec) -> ClaudeAgentOptions: environment["CLAUDE_CODE_DISABLE_BACKGROUND_TASKS"] = "1" allowed.extend(DELEGATION_TOOLS) _can_reach_its_workers(spec, allowed) - builtins = [name for name in spec.builtins if name != DELEGATE_TOOL] + # Workers draw their built-ins from this session's set, so it is the union of all of them. + builtins = [ + name + for name in dict.fromkeys( + [*spec.builtins, *(n for w in spec.workers.values() for n in (w.builtins or spec.builtins))] + ) + if name != DELEGATE_TOOL + ] options = ClaudeAgentOptions( tools=[*builtins, *(DELEGATION_TOOLS if spec.workers else ())], system_prompt=_prompt_for(spec.system_prompt), diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 7e83b4450..23605bb21 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -1545,17 +1545,18 @@ async def _read_platform_results(self, _args: dict[str, Any]) -> dict[str, Any]: async def _ask( self, _tool_name: str, payload: dict[str, Any], _context: Any ) -> Any: - if self._from_platform: - return { - "content": [ - { - "type": "text", - "text": "Nobody answers during the build or its check. Decide it yourself from " - "the agent and the contract, and say what you decided in your summary.", - } - ] - } asked = [one for one in payload.get("questions") or [] if isinstance(one, dict)] + if self._from_platform: + nobody = ( + "Nobody answers during the build or its check. Decide it yourself from " + "the agent and the contract, and say what you decided in your summary." + ) + if asked and self.backend_name == "claude": + # The CLI's own question tool arrives as a permission check, which needs a decision. + from claude_agent_sdk.types import PermissionResultDeny + + return PermissionResultDeny(message=nobody) + return {"content": [{"type": "text", "text": nobody}]} question = str( payload.get("question") or "\n".join(str(one.get("question") or "") for one in asked if one.get("question")) diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index d57c86791..4095a9905 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -664,4 +664,7 @@ async def handler(args): assert asyncio.run(drop.handler({"names": ["one"]}))["is_error"] is True answer = asyncio.run(runtime._ask("AskUserQuestion", {"question": "Which?"}, None)) assert "Nobody answers" in answer["content"][0]["text"] + runtime.backend_name = "claude" + denied = asyncio.run(runtime._ask("AskUserQuestion", {"questions": [{"question": "Which?"}]}, None)) + assert type(denied).__name__ == "PermissionResultDeny" and "Nobody answers" in denied.message assert calls == [corrected] diff --git a/tests/test_harness.py b/tests/test_harness.py index b243cfec1..d72ddd25b 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -4444,6 +4444,40 @@ def test_hosted_claude_session_uses_durable_resume_and_restricted_tools(tmp_path assert options.env["CLAUDE_CONFIG_DIR"] == str(tmp_path / ".claude") +def test_claude_session_offers_its_workers_builtins_and_routes_questions_through_ask(monkeypatch): + import asyncio + + from claude_agent_sdk.types import PermissionResultDeny + + from fi.alk.harness.backends import SessionSpec, WorkerSpec + from fi.alk.harness.backends.claude import ClaudeBackend + + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_URL", "https://gateway.futureagi.test") + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_API_KEY", "internal-key") + asked = [] + + async def ask(name, payload, context): + asked.append(name) + return "answered" + + planner = SessionSpec( + system_prompt="plan", + builtins=("AskUserQuestion", "Delegate"), + model="vertex_ai/gemini-3.7-flash", + workers={"writer": WorkerSpec(description="w", instructions="w", builtins=("Read", "Glob", "Grep"))}, + ask=ask, + ) + options = ClaudeBackend().create(planner)._options + assert {"Read", "Glob", "Grep", "AskUserQuestion", "Agent", "Task"} <= set(options.tools) + assert "Delegate" not in options.tools and "AskUserQuestion" not in options.allowed_tools + assert asyncio.run(options.can_use_tool("AskUserQuestion", {"questions": []}, None)) == "answered" + assert asked == ["AskUserQuestion"] + + unattended = SessionSpec(system_prompt="s", builtins=("AskUserQuestion",), model="vertex_ai/gemini-3.7-flash") + gate = ClaudeBackend().create(unattended)._options.can_use_tool + assert isinstance(asyncio.run(gate("AskUserQuestion", {"questions": []}, None)), PermissionResultDeny) + + def test_claude_backend_routes_gemini_through_agentcc(monkeypatch): from fi.alk.harness.backends import SessionSpec from fi.alk.harness.backends.claude import ClaudeBackend From 0251563709f27303caed1b5cbe02c0402454b334 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 07:46:09 +0530 Subject: [PATCH 11/12] test(harness): read the skills' sub-agent, verifier and calling-number names from where they are defined --- tests/test_harness.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_harness.py b/tests/test_harness.py index d72ddd25b..4f68e9813 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -4692,9 +4692,16 @@ def test_a_skill_only_names_tools_its_stage_actually_has(): _SILENT_ENVIRONMENTS, ) from fi.alk.harness.scenario import _AXIS_LABELS, _LEVEL_LABELS + from fi.alk.harness.catalogue import CONTRACT_SCHEMA + from fi.alk.harness.simulator_voice import _CALLER_PHONE_KEYS + from fi.alk.harness.understand import READER ignore = ( fields + # A sub-agent, a verifier value and the fixture key for the calling number, not tools. + | {READER} + | set(CONTRACT_SCHEMA["verifier"]["enum"]) + | set(_CALLER_PHONE_KEYS) | set(MODALITIES) | set(CALL_DIRECTIONS) | set(CALLER_AWARENESS) From ef5b4d413b14e6a4f435e7a5c613a4b7bf0ca3bf Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 17:07:26 +0530 Subject: [PATCH 12/12] ci: run the harness tests on every pull request --- .github/workflows/harness-tests.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/harness-tests.yml diff --git a/.github/workflows/harness-tests.yml b/.github/workflows/harness-tests.yml new file mode 100644 index 000000000..1d2383e3b --- /dev/null +++ b/.github/workflows/harness-tests.yml @@ -0,0 +1,23 @@ +name: Harness tests + +permissions: + contents: read + +on: + push: + branches: [dev, main] + pull_request: + +jobs: + harness: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + - name: Install + run: uv sync --extra livekit --extra harness-stores --extra harness-ui --extra hosted-tracing + - name: Run harness tests + run: uv run pytest -q -p no:cacheprovider tests/test_harness.py tests/harness