Skip to content
Merged
2 changes: 1 addition & 1 deletion .github/workflows/sdk-smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
- name: Install test dependencies
# Exercise the minimum LiveKit version supported by pyproject.toml.
run: >-
python -m pip install '.[livekit]' pytest pytest-asyncio
python -m pip install '.[livekit]' pytest pytest-asyncio 'hypothesis>=6.140'
'livekit-agents==1.8.3'
'livekit-plugins-cartesia==1.8.3'
'livekit-plugins-deepgram==1.8.3'
Expand Down
70 changes: 30 additions & 40 deletions src/fi/alk/harness/amend.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,18 @@

from __future__ import annotations

import json
from pathlib import Path

from .contract import AgentContract, known_modalities, validate_contract
from ..providers.agent import _redact
from .contract import (
CONTRACT_FILE,
AgentContract,
known_modalities,
validate_contract,
write_contract,
)

CONTRACT = "contract.json"
CONTRACT = CONTRACT_FILE


def widen(
Expand Down Expand Up @@ -78,12 +84,7 @@ def widen(
contract.amendments.pop()
return False, "the amended contract would not be valid: " + "; ".join(problems)

destination = Path(destination)
destination.mkdir(parents=True, exist_ok=True)
(destination / CONTRACT).write_text(
json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
encoding="utf-8",
)
_persist(contract, destination)
return True, (
f"{tool_name}.{argument} now accepts {', '.join(fresh)}. "
f"{len(contract.amendments)} amendment(s) recorded on the contract."
Expand All @@ -104,8 +105,12 @@ def add_rule(
return False, "no rule given"
if not why.strip():
return False, "say why: an unexplained rule is indistinguishable from a guess"
if any(rule.lower() == existing.lower() for existing in contract.hard_constraints):
return False, f"the agent already has that rule: {rule}"
# Both sides are compared as they are saved: a contract held from an older save can still
# hold its rules raw. One call, so a refused URL is logged once. The reply names the rule as
# saved too, since the chat sends it on to the platform.
cleaned, *existing = _redact([rule, *contract.hard_constraints])
if cleaned.lower() in (one.lower() for one in existing):
return False, f"the agent already has that rule: {cleaned}"

contract.hard_constraints.append(rule)
contract.amendments.append(f"rule added — {rule}: {why.strip()}")
Expand All @@ -115,12 +120,7 @@ def add_rule(
contract.amendments.pop()
return False, "the amended contract would not be valid: " + "; ".join(problems)

destination = Path(destination)
destination.mkdir(parents=True, exist_ok=True)
(destination / CONTRACT).write_text(
json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
encoding="utf-8",
)
_persist(contract, destination)
return True, (
f"added. The agent now has {len(contract.hard_constraints)} rules, and this one is "
"graded from here on."
Expand Down Expand Up @@ -166,12 +166,7 @@ def set_modality(
contract.amendments.pop()
return False, "the amended contract would not be valid: " + "; ".join(problems)

destination = Path(destination)
destination.mkdir(parents=True, exist_ok=True)
(destination / CONTRACT).write_text(
json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
encoding="utf-8",
)
_persist(contract, destination)
reached = {
"voice": "a call placed to the agent where it is hosted, its own tools answered over a "
"webhook",
Expand All @@ -191,25 +186,23 @@ def drop_rule(
"""
if not why.strip():
return False, "say why: removing a rule changes what is being graded"
# Both sides are compared as they are saved: a contract held from an older save can still
# hold its rules raw. One call, so a refused URL is logged once.
wanted, *saved_rules = _redact([rule.strip(), *contract.hard_constraints])
wanted = wanted.lower()
held = list(zip(contract.hard_constraints, (one.lower() for one in saved_rules)))
match = next(
(
existing
for existing in contract.hard_constraints
if existing.lower() == rule.strip().lower()
),
(existing for existing, as_saved in held if as_saved == wanted),
None,
) or next(
(
existing
for existing in contract.hard_constraints
if rule.strip().lower() in existing.lower()
),
(existing for existing, as_saved in held if wanted in as_saved),
None,
)
if match is None:
# The rules as saved, since the chat sends this reply on to the platform.
return False, (
"no rule like that. It has:\n - "
+ "\n - ".join(contract.hard_constraints)
+ "\n - ".join(saved_rules)
)
contract.hard_constraints.remove(match)
contract.amendments.append(f"rule removed — {match}: {why.strip()}")
Expand Down Expand Up @@ -322,12 +315,9 @@ def drop_tool_references(contract: AgentContract) -> list[str]:


def _persist(contract: AgentContract, destination: Path) -> None:
destination = Path(destination)
destination.mkdir(parents=True, exist_ok=True)
(destination / CONTRACT).write_text(
json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
encoding="utf-8",
)
"""Save the contract the stage is holding. Its free text is cleaned in place as it is saved,
so the stage goes on working from what was saved."""
write_contract(contract, Path(destination))


def not_offered(contract: AgentContract, candidates: dict[str, set[str]]) -> list[str]:
Expand Down
40 changes: 24 additions & 16 deletions src/fi/alk/harness/authoring_runtime_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1427,6 +1427,27 @@ def _invariants_the_ir_still_breaks(world, authoring: Path) -> list[str]:
return []


def _promote_data_store(contract, authoring: Path, candidate_dir: Path) -> int:
"""Copy only ``data_store`` from a candidate reading onto the saved contract, and save it.

Returns 0 when it was promoted, 1 when the candidate has none to give.
"""
from .contract import AgentContract, write_contract

candidate_path = candidate_dir / "contract.json"
if not candidate_path.is_file():
return 1
candidate = AgentContract.model_validate_json(
candidate_path.read_text(encoding="utf-8")
)
if candidate.data_store is None or not candidate.data_store.kind.strip():
return 1
payload = contract.model_dump()
payload["data_store"] = candidate.data_store.model_dump()
write_contract(AgentContract.model_validate(payload), authoring)
return 0


async def validate_and_repair(
job,
source: Path,
Expand Down Expand Up @@ -1509,24 +1530,11 @@ async def repair(phase, guidance):
],
)
)
candidate_path = Path(candidate_dir) / "contract.json"
if status != 0 or not candidate_path.is_file():
return 1
candidate = AgentContract.model_validate_json(
candidate_path.read_text(encoding="utf-8")
)
if (
candidate.data_store is None
or not candidate.data_store.kind.strip()
):
if status != 0:
return 1
payload = contract.model_dump()
payload["data_store"] = candidate.data_store.model_dump()
repaired = AgentContract.model_validate(payload)
contract_path.write_text(
repaired.model_dump_json(indent=2) + "\n", encoding="utf-8"
return _promote_data_store(
contract, authoring, Path(candidate_dir)
)
return 0
if generic and _needs_runtime_plan_repair(_active_repair_diagnostics):
from .contract import AgentContract
from .runtime_repair import request_runtime_plan_patch
Expand Down
16 changes: 13 additions & 3 deletions src/fi/alk/harness/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from pathlib import Path
from typing import Any

from ..providers.agent import _redact
from . import spend
from . import observability
from .build import open_stage as build_stage
Expand Down Expand Up @@ -62,7 +63,9 @@ def _source_root(destination: Path, explicit: str = "") -> str:


def _render(event: Event) -> None:
line = event.line()
# The platform keeps the tail of this output, so a URL's password, query and fragment are
# dropped first, in one call per event, as emit_model_event drops them from the job's events.
line = _redact(event.line())
if event.kind == TEXT:
print(line, end="", flush=True)
else:
Expand Down Expand Up @@ -109,7 +112,11 @@ def receive(event: Event) -> None:
emitted.append(event.text)

await stage.say(content, on_event=receive)
text = "".join(emitted).strip() or "I’m still working on the active authoring stage."
# Sent to the platform's chat as the hosted builder's replies are, so cleaned the same way.
text = (
_redact("".join(emitted).strip())
or "I’m still working on the active authoring stage."
)
await client.emit(
"assistant_message",
message_id=str(uuid.uuid4()),
Expand Down Expand Up @@ -1015,12 +1022,15 @@ def emit_model_event(event: Event) -> None:
for key in ("target", "label", "call_id", "is_error", "path", "outcome", "cost_usd")
if key in detail
}
# The job response carries these, so a URL's password or query is dropped first, in
# one call, before the text is cut.
text, safe_detail = _redact([event.text or None, safe_detail])
emit(
"harness.activity",
str(detail.get("stage") or "authoring"),
event_kind=event.kind,
tool=event.tool or None,
text=event.text[:2000] if event.text else None,
text=text[:2000] if text else None,
detail=safe_detail,
)

Expand Down
Loading
Loading