Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
736b23a
test(harness): drop a removed tool's test body that a merge left insi…
KarthikAvinashFI Oct 22, 2026
678b9aa
test(harness): give the dispatch-ack fakes the failure, report and ru…
KarthikAvinashFI Oct 22, 2026
d0e9cbe
fix(harness): route the ask tool through the permission callback and …
KarthikAvinashFI Oct 22, 2026
d64b55e
test(harness): name the scenario fixture fields as the model has them…
KarthikAvinashFI Oct 22, 2026
d2b990e
test(harness): set the prompt budget from the skills as they now stand
KarthikAvinashFI Oct 22, 2026
7156209
test(simulator): match the caller's re-ask rule to its current wording
KarthikAvinashFI Oct 22, 2026
2464f3a
test(harness): give hosted job fixtures the pinned commit, isolation …
KarthikAvinashFI Oct 22, 2026
9497fd6
docs(harness): describe the proof gates without naming another benchmark
KarthikAvinashFI Oct 22, 2026
166adb3
test(studio): point the stub account at the URL variable the client r…
KarthikAvinashFI Oct 22, 2026
39b0a86
Merge remote-tracking branch 'origin/dev' into fix/alk-dev-red-tests
KarthikAvinashFI Oct 22, 2026
882a4e4
Merge remote-tracking branch 'origin/dev' into fix/alk-dev-red-tests
KarthikAvinashFI Oct 22, 2026
61ae259
fix(harness): answer the CLI's question tool with a decision during p…
KarthikAvinashFI Oct 22, 2026
0251563
test(harness): read the skills' sub-agent, verifier and calling-numbe…
KarthikAvinashFI Oct 23, 2026
9284a77
Merge branch 'dev' of https://github.com/future-agi/agent-learning-ki…
KarthikAvinashFI Oct 23, 2026
ef5b4d4
ci: run the harness tests on every pull request
KarthikAvinashFI Oct 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions .github/workflows/harness-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: Harness tests

permissions:
contents: read

on:
push:
branches: [dev, main]
pull_request:

jobs:
harness:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
python-version: "3.12"
- name: Install
run: uv sync --extra livekit --extra harness-stores --extra harness-ui --extra hosted-tracing
- name: Run harness tests
run: uv run pytest -q -p no:cacheprovider tests/test_harness.py tests/harness
2 changes: 1 addition & 1 deletion src/fi/alk/harness/DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ was invented, tone.

## 4. Three gates on every scenario, before it is kept

Terminal-bench's oracle run, which is the reason its tasks are known to be solvable.
A reference run like this is how a benchmark's tasks are known to be solvable.

### Gate 1. Ready

Expand Down
13 changes: 12 additions & 1 deletion src/fi/alk/harness/backends/claude.py
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,9 @@ def _options(self, spec: SessionSpec) -> ClaudeAgentOptions:
wire_model = gateway_wire_model(spec.model)
gateway_compatible = behind_gateway(spec.model)
# A worker's calls are made inside this session, so the gate covers its tools too.
allowed = [name for name in spec.granted_anywhere() if name != DELEGATE_TOOL]
# The ask tool stays out: an allowed tool is approved before the permission callback, so the
# question would never reach the person.
allowed = [name for name in spec.granted_anywhere() if name not in (DELEGATE_TOOL, ASK_TOOL)]
# Union per server name; each agent is still restricted by its own `tools` allowlist.
servers: dict[str, ToolServer] = {}
for source in (spec.servers, *(worker.servers for worker in spec.workers.values())):
Expand Down Expand Up @@ -431,7 +433,16 @@ def _options(self, spec: SessionSpec) -> ClaudeAgentOptions:
environment["CLAUDE_CODE_DISABLE_BACKGROUND_TASKS"] = "1"
allowed.extend(DELEGATION_TOOLS)
_can_reach_its_workers(spec, allowed)
# Workers draw their built-ins from this session's set, so it is the union of all of them.
builtins = [
name
for name in dict.fromkeys(
[*spec.builtins, *(n for w in spec.workers.values() for n in (w.builtins or spec.builtins))]
)
if name != DELEGATE_TOOL
]
options = ClaudeAgentOptions(
tools=[*builtins, *(DELEGATION_TOOLS if spec.workers else ())],
system_prompt=_prompt_for(spec.system_prompt),
allowed_tools=allowed,
mcp_servers={
Expand Down
21 changes: 11 additions & 10 deletions src/fi/alk/harness/hosted_chat_entrypoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -1545,17 +1545,18 @@ async def _read_platform_results(self, _args: dict[str, Any]) -> dict[str, Any]:
async def _ask(
self, _tool_name: str, payload: dict[str, Any], _context: Any
) -> Any:
if self._from_platform:
return {
"content": [
{
"type": "text",
"text": "Nobody answers during the build or its check. Decide it yourself from "
"the agent and the contract, and say what you decided in your summary.",
}
]
}
asked = [one for one in payload.get("questions") or [] if isinstance(one, dict)]
if self._from_platform:
nobody = (
"Nobody answers during the build or its check. Decide it yourself from "
"the agent and the contract, and say what you decided in your summary."
)
if asked and self.backend_name == "claude":
# The CLI's own question tool arrives as a permission check, which needs a decision.
from claude_agent_sdk.types import PermissionResultDeny

return PermissionResultDeny(message=nobody)
return {"content": [{"type": "text", "text": nobody}]}
question = str(
payload.get("question")
or "\n".join(str(one.get("question") or "") for one in asked if one.get("question"))
Expand Down
3 changes: 1 addition & 2 deletions src/fi/alk/harness/prove.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
"""Proving a scenario is worth keeping, before anything is ever run against the agent.

Three gates, all pure code. No model is asked whether a scenario is good; the environment
decides. Terminal-bench keeps its tasks honest this way, and it is the cheapest useful thing in
the whole harness: no tokens, no network, a few milliseconds.
decides. It is the cheapest useful thing in the whole harness: no tokens, no network, a few milliseconds.

**Ready.** Reset the world, run the scenario's own ``setup.py``, then its ``ready.py``. The world
has to hold what the scenario presumes. A scenario about the last five chocolates is only a test
Expand Down
3 changes: 1 addition & 2 deletions src/fi/alk/harness/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@

It also carries a **solution**: what a correct agent would do. That is not decoration. It is what
proves, before the scenario is ever used, that the scenario can be passed at all and that its
checks are not vacuous — the two gates in ``prove.py``. Terminal-bench keeps its tasks honest the
same way, and it needs no model to do it.
checks are not vacuous — the two gates in ``prove.py``. It needs no model to do it.
"""

from __future__ import annotations
Expand Down
2 changes: 1 addition & 1 deletion src/fi/alk/harness/skills/write-scenarios/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -685,7 +685,7 @@ conversation is the only evidence there will be. Everything else in this skill s
- **Take the difficulty from the agent's instructions.** Its policies, required questions,
disclosures, refusals, escalation rules and limits are the cells. A tool that fails or an account
state are not available here.
- `setup`/`ready` only if the caller needs a private fixture; usually both stay empty.
- `setup_code`/`ready_code` only if the caller needs a private fixture; usually both stay empty.

## Before you submit: read it the way a reviewer will

Expand Down
3 changes: 3 additions & 0 deletions tests/harness/test_builder_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -664,4 +664,7 @@ async def handler(args):
assert asyncio.run(drop.handler({"names": ["one"]}))["is_error"] is True
answer = asyncio.run(runtime._ask("AskUserQuestion", {"question": "Which?"}, None))
assert "Nobody answers" in answer["content"][0]["text"]
runtime.backend_name = "claude"
denied = asyncio.run(runtime._ask("AskUserQuestion", {"questions": [{"question": "Which?"}]}, None))
assert type(denied).__name__ == "PermissionResultDeny" and "Nobody answers" in denied.message
assert calls == [corrected]
12 changes: 10 additions & 2 deletions tests/harness/test_dispatch_ack.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from fi.alk.harness import hosted_scheduler as hs
from fi.alk.harness.hosted_scheduler import CallAborted
from fi.alk.harness.job import FailureDomain
from fi.simulate.runtime.failures import FailureStage, SimulationFailure
from fi.simulate.runtime.run import TestCaseStatus

_UNACK = "voice_dispatch_unacknowledged"
Expand Down Expand Up @@ -125,19 +126,26 @@ def _runner() -> cr.CallRunnerImpl:
runner = cr.CallRunnerImpl.__new__(cr.CallRunnerImpl)
runner._adapter = None # never touched: no calls, no result
runner._collect_calls = lambda runtime: () # instance override
runner._context = SimpleNamespace(usage_reporter=None)
runner._environ = {}
return runner


def _report(*, status: TestCaseStatus, code: str | None) -> SimpleNamespace:
failure = SimpleNamespace(code=code, message="dispatch not acked") if code else None
failure = (
SimulationFailure(stage=FailureStage.RUNNING, code=code, message="dispatch not acked")
if code
else None
)
case = SimpleNamespace(
test_case_id="c1",
status=status,
started_at=None,
ended_at=None,
result=None,
failure=failure,
)
return SimpleNamespace(test_cases=[case])
return SimpleNamespace(run_id="r1", test_cases=[case])


def _translate(report: SimpleNamespace) -> CallAborted:
Expand Down
40 changes: 0 additions & 40 deletions tests/harness/test_grade.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,46 +230,6 @@ def test_the_cap_holds_for_the_session_that_saves_too(tmp_path):
said = asyncio.run(submit.handler({"name": "one_more", "instruction": "three"}))
assert said.get("is_error"), can_save
assert "2 of 2 written" in said["content"][0]["text"]
contract = AgentContract(
agent="cart", real_use_cases=["add an item", "remove an item"]
)
# Above FEWEST_WORTH_DELEGATING, or the fan-out tool is not published at all.
server, _kept = st.scenario_tools(contract, tmp_path, tmp_path, wanted=20)
suite = next(spec for spec in server.tools if spec.name == "generate_suite")

asked_for: list[int] = []

async def _fake_parallel(contract, *, out, wanted, use_cases, slices, at_once):
asked_for.append(wanted)
return []

monkeypatch.setattr("fi.alk.harness.scenarios.write_in_parallel", _fake_parallel)
# Nothing on disk yet: the full ask goes through.
asyncio.run(suite.handler({"count": 20}))
assert asked_for == [20]

# Fourteen already written, and the model asks for twenty again: six are outstanding.
monkeypatch.setattr(
st,
"load_scenarios",
lambda _d: [
Scenario(name=f"s{i}", instruction="i", sub_goals=["x"]) for i in range(14)
],
)
monkeypatch.setattr(st, "journalled", lambda _d: [])
asyncio.run(suite.handler({"count": 20}))
assert asked_for == [20, 6]

monkeypatch.setattr(
st,
"load_scenarios",
lambda _d: [
Scenario(name=f"s{i}", instruction="i", sub_goals=["x"]) for i in range(20)
],
)
said = asyncio.run(suite.handler({"count": 20}))
assert "already holds the 20" in said["content"][0]["text"]
assert asked_for == [20, 6]


def test_only_chat_can_grow_a_finished_suite(tmp_path):
Expand Down
55 changes: 51 additions & 4 deletions tests/test_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -4405,16 +4405,18 @@ async def hand(_args):
assert denied != {}


def test_hosted_claude_session_uses_durable_resume_and_restricted_tools(tmp_path):
def test_hosted_claude_session_uses_durable_resume_and_restricted_tools(tmp_path, monkeypatch):
from fi.alk.harness.backends import ConversationSession, SessionSpec
from fi.alk.harness.backends.claude import ClaudeBackend

monkeypatch.setenv("ALK_CLAUDE_GATEWAY_URL", "https://gateway.futureagi.test")
monkeypatch.setenv("ALK_CLAUDE_GATEWAY_API_KEY", "internal-key")
transcript_store = object()
spec = SessionSpec(
system_prompt="hosted",
builtins=("Read", "AskUserQuestion"),
cwd=str(tmp_path),
model="claude-sonnet-4-6",
model="vertex_ai/gemini-3.7-flash",
conversation=ConversationSession(
app_name="alk-harness",
user_id="conversation-user",
Expand Down Expand Up @@ -4442,6 +4444,40 @@ def test_hosted_claude_session_uses_durable_resume_and_restricted_tools(tmp_path
assert options.env["CLAUDE_CONFIG_DIR"] == str(tmp_path / ".claude")


def test_claude_session_offers_its_workers_builtins_and_routes_questions_through_ask(monkeypatch):
import asyncio

from claude_agent_sdk.types import PermissionResultDeny

from fi.alk.harness.backends import SessionSpec, WorkerSpec
from fi.alk.harness.backends.claude import ClaudeBackend

monkeypatch.setenv("ALK_CLAUDE_GATEWAY_URL", "https://gateway.futureagi.test")
monkeypatch.setenv("ALK_CLAUDE_GATEWAY_API_KEY", "internal-key")
asked = []

async def ask(name, payload, context):
asked.append(name)
return "answered"

planner = SessionSpec(
system_prompt="plan",
builtins=("AskUserQuestion", "Delegate"),
model="vertex_ai/gemini-3.7-flash",
workers={"writer": WorkerSpec(description="w", instructions="w", builtins=("Read", "Glob", "Grep"))},
ask=ask,
)
options = ClaudeBackend().create(planner)._options
assert {"Read", "Glob", "Grep", "AskUserQuestion", "Agent", "Task"} <= set(options.tools)
assert "Delegate" not in options.tools and "AskUserQuestion" not in options.allowed_tools
assert asyncio.run(options.can_use_tool("AskUserQuestion", {"questions": []}, None)) == "answered"
assert asked == ["AskUserQuestion"]

unattended = SessionSpec(system_prompt="s", builtins=("AskUserQuestion",), model="vertex_ai/gemini-3.7-flash")
gate = ClaudeBackend().create(unattended)._options.can_use_tool
assert isinstance(asyncio.run(gate("AskUserQuestion", {"questions": []}, None)), PermissionResultDeny)


def test_claude_backend_routes_gemini_through_agentcc(monkeypatch):
from fi.alk.harness.backends import SessionSpec
from fi.alk.harness.backends.claude import ClaudeBackend
Expand Down Expand Up @@ -4656,9 +4692,16 @@ def test_a_skill_only_names_tools_its_stage_actually_has():
_SILENT_ENVIRONMENTS,
)
from fi.alk.harness.scenario import _AXIS_LABELS, _LEVEL_LABELS
from fi.alk.harness.catalogue import CONTRACT_SCHEMA
from fi.alk.harness.simulator_voice import _CALLER_PHONE_KEYS
from fi.alk.harness.understand import READER

ignore = (
fields
# A sub-agent, a verifier value and the fixture key for the calling number, not tools.
| {READER}
| set(CONTRACT_SCHEMA["verifier"]["enum"])
| set(_CALLER_PHONE_KEYS)
| set(MODALITIES)
| set(CALL_DIRECTIONS)
| set(CALLER_AWARENESS)
Expand All @@ -4671,6 +4714,9 @@ def test_a_skill_only_names_tools_its_stage_actually_has():
| {"origin", "matching"}
| {"handle", "check", "args", "db", "world", "calls", "json", "ToolError"}
| {"book_ride", "cancel_ride", "verify_otp", "bookings"}
| {"create_booking", "cancel_booking", "verify_code", "cancel_subscription", "reset_password"}
# Words a check example searches for or reads, not tools.
| {"company", "pan"}
| {"social_engineering_resisted", "true", "or"}
)

Expand Down Expand Up @@ -7738,8 +7784,9 @@ def test_the_system_prompt_stays_within_budget():
contract.brief(with_data=True, sample_rows=3) + load_skill("write-scenarios") + kinds
)
main_loop = sub_agent + load_skill("plan-suite", preamble=False)
assert len(sub_agent) <= 90_000, f"sub-agent prompt is {len(sub_agent)} chars"
assert len(main_loop) <= 130_000, f"main loop prompt is {len(main_loop)} chars"
# Set from the skills as they stand plus about a tenth: growth past this is a choice to make, not drift.
assert len(sub_agent) <= 165_000, f"sub-agent prompt is {len(sub_agent)} chars"
assert len(main_loop) <= 245_000, f"main loop prompt is {len(main_loop)} chars"


def test_a_sub_goal_defined_after_the_fact_reaches_the_folders_already_written(tmp_path):
Expand Down
Loading
Loading