Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
1f1da64
fix(harness): save whether each proof's solution ran and report prove…
KarthikAvinashFI Nov 14, 2026
70e7401
feat(harness): negative controls, leaked-answer check, grounding and …
KarthikAvinashFI Nov 14, 2026
7281e33
fix(harness): read compared literals inside a check's return conditio…
KarthikAvinashFI Nov 14, 2026
dd4e8e5
test(harness): pin that a raised failure message is not a grader-only…
KarthikAvinashFI Nov 14, 2026
36d58c7
fix(harness): an assumed solution step never reads as a no-change cas…
KarthikAvinashFI Nov 14, 2026
b9b21ee
fix(harness): append code-written proof counts to the finishing reply…
KarthikAvinashFI Nov 14, 2026
32818d1
Merge branch 'feat/TH-8752-proof-honesty' into feat/TH-8918-scenario-…
KarthikAvinashFI Nov 14, 2026
ad37f0c
feat(harness): a judged-only case is proved only when the judge fails…
KarthikAvinashFI Nov 14, 2026
445444e
feat(harness): prove_suite proves only the scenario keys passed with …
KarthikAvinashFI Nov 14, 2026
bf2c058
feat(harness): compare the reference solution's argument values as gr…
KarthikAvinashFI Nov 14, 2026
e61ebba
feat(harness): judge each cited rule against the expected outcome and…
KarthikAvinashFI Nov 14, 2026
ec69f38
feat(harness): the builder refuses a new or changed scenario with no …
KarthikAvinashFI Nov 14, 2026
f76c3f2
feat(harness): a scenario carries must_pass, versioned with its content
KarthikAvinashFI Nov 14, 2026
23010c0
docs(harness): tell the scenario writer when to mark a case must_pass
KarthikAvinashFI Nov 14, 2026
6d08f42
feat(harness): record what each check reads, tool calls or world stat…
KarthikAvinashFI Nov 14, 2026
d751967
feat(harness): prove reference solutions and negative controls on the…
KarthikAvinashFI Nov 14, 2026
c12abcf
fix(harness): a scenario a run cannot load is left out of the seal an…
KarthikAvinashFI Nov 14, 2026
41fbf93
feat(harness): record why a rebuild drops a scenario for a removed ca…
KarthikAvinashFI Nov 14, 2026
1f150a0
fix(harness): send at most the platform's 200 failed scenarios with a…
KarthikAvinashFI Nov 14, 2026
70e8de2
fix(harness): a check that raises under a negative control reads brok…
KarthikAvinashFI Nov 14, 2026
d45eba3
fix(harness): a reference amount written as 37.5 or 37.50, or a value…
KarthikAvinashFI Nov 14, 2026
2a29511
fix(harness): a blank assumption source does not count as citing wher…
KarthikAvinashFI Nov 14, 2026
b90d51d
Merge branch 'feat/TH-8918-scenario-lifecycle' into feat/TH-8754-rest…
KarthikAvinashFI Nov 15, 2026
8716adc
Merge branch 'feat/TH-8918-scenario-lifecycle' into fix/TH-8929-parti…
KarthikAvinashFI Nov 15, 2026
20d549a
test(harness): with no drops left dropped.json is removed and the sce…
KarthikAvinashFI Nov 15, 2026
0583072
fix(harness): a held scenario's last sealed copy never takes the fold…
KarthikAvinashFI Nov 15, 2026
e448180
test(harness): builder-submitted cases in the fixtures carry an assum…
KarthikAvinashFI Nov 15, 2026
3b2fbf9
test(harness): the builder's submit refuses a new case that cites not…
KarthikAvinashFI Nov 15, 2026
00e73fb
Merge branch 'feat/TH-8918-scenario-lifecycle' into fix/TH-8929-parti…
KarthikAvinashFI Nov 15, 2026
6488420
Merge branch 'feat/TH-8918-scenario-lifecycle' into feat/TH-8754-rest…
KarthikAvinashFI Nov 15, 2026
99122c5
test(harness): the proof stub returns the scenario-code outcome and t…
KarthikAvinashFI Nov 15, 2026
271c543
Merge branch 'feat/TH-8752-proof-honesty' into feat/TH-8918-scenario-…
KarthikAvinashFI Nov 15, 2026
6018077
Merge branch 'feat/TH-8918-scenario-lifecycle' into feat/TH-8754-rest…
KarthikAvinashFI Nov 15, 2026
147ee53
Merge branch 'feat/TH-8918-scenario-lifecycle' into fix/TH-8929-parti…
KarthikAvinashFI Nov 15, 2026
d333f0c
fix(harness): write must_pass into a scenario only when it is set, so…
KarthikAvinashFI Nov 15, 2026
bd49f78
Merge branch 'feat/TH-8918-scenario-lifecycle' into feat/TH-8754-rest…
KarthikAvinashFI Nov 15, 2026
f44b54e
Merge branch 'feat/TH-8918-scenario-lifecycle' into fix/TH-8929-parti…
KarthikAvinashFI Nov 15, 2026
6787e22
fix(harness): a rebuild adds back a tool the agent defines again with…
KarthikAvinashFI Nov 15, 2026
0ec264e
chore: merge feat/TH-8754-restore-dropped-scenarios into feat/TH-8918…
KarthikAvinashFI Nov 15, 2026
eba3440
fix(harness): a re-certify seeds the environment's own World IR besid…
KarthikAvinashFI Nov 15, 2026
a82e465
fix(harness): the generated MCP Toolbox server answers a client that …
KarthikAvinashFI Nov 15, 2026
dc37ad0
fix(harness): a builder save tidies keywords only on the cases it wrote
KarthikAvinashFI Nov 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions src/fi/alk/harness/amend.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@ def widen(
spec = next((tool for tool in contract.tools if tool.name == tool_name), None)
if spec is None:
return False, (
f"{tool_name!r} is not a tool this agent has. It has: "
f"{', '.join(sorted(contract.tool_names()))}"
f"{tool_name!r} is not in the contract. It has: "
f"{', '.join(sorted(contract.tool_names()))}. To add a tool the agent's source defines "
"(one it gained, or one an earlier version removed), use declare_tool."
)
if argument not in spec.args:
return False, (
Expand Down Expand Up @@ -242,8 +243,9 @@ def fix_tool(
spec = next((tool for tool in contract.tools if tool.name == tool_name), None)
if spec is None:
return False, (
f"{tool_name!r} is not a tool this agent has. It has: "
f"{', '.join(sorted(contract.tool_names()))}"
f"{tool_name!r} is not in the contract. It has: "
f"{', '.join(sorted(contract.tool_names()))}. To add a tool the agent's source defines "
"(one it gained, or one an earlier version removed), use declare_tool."
)

if remove:
Expand Down
26 changes: 26 additions & 0 deletions src/fi/alk/harness/authoring_entrypoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,8 @@ async def refuse_repair(phase: str, guidance: str) -> int:
raise RuntimeValidationError(
"environment", f"Certify-only validation failed: {_one_line(exc)}"
) from exc
if getattr(args, "prove_scenarios", False):
_prove_scenarios(output, getattr(args, "deadline_at", None))
try:
# Written before sealing: the platform's fill gate keys on the sealed certificate.
report_path.write_text(
Expand Down Expand Up @@ -458,6 +460,19 @@ async def refuse_repair(phase: str, guidance: str) -> int:
return 0


SCENARIO_PROOFS = "scenario-proofs.json"


def _prove_scenarios(output: Path, deadline_at: float | None) -> None:
"""Prove every scenario on this certification's authoring tree; never fails the certify."""
from .prove_suite import prove_suite

try:
prove_suite(output, output / SCENARIO_PROOFS, deadline_at=deadline_at)
except Exception as exc: # noqa: BLE001 - a missing proofs file reads as "no new proofs"
print(f"certify-only: scenario proofs not written: {_one_line(exc)}", file=sys.stderr, flush=True)


def _load_provider_import_profile(
job: HarnessJob,
secrets_path: Path | None,
Expand Down Expand Up @@ -674,6 +689,17 @@ def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> in
type=Path,
help="Directory the sealed bundle is written to in --certify-only mode",
)
parser.add_argument(
"--prove-scenarios",
action="store_true",
help="With --certify-only: prove every scenario after validation and before sealing",
)
parser.add_argument(
"--deadline-at",
type=float,
default=None,
help="Epoch seconds the validation job is stopped at; scenario proving keeps a margin before it",
)
args = parser.parse_args(argv)
if args.certify_only:
if args.bundle_output is None:
Expand Down
21 changes: 21 additions & 0 deletions src/fi/alk/harness/authoring_runtime_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1333,6 +1333,27 @@ async def check_setup_once(scenario, invariants, against, keep_processes):
f"invariants, complete in {time.monotonic() - started:.0f}s",
flush=True,
)
if not readiness_only:
# While the stack is up, play each case's reference and controls through the
# service-backed tools for real; never fails validation.
from .live_proof import prove_live

try:
live = await prove_live(
authoring=authoring,
contract=AgentContract.model_validate_json(
(bundle / "contract.json").read_text(encoding="utf-8")
),
runtime=runtime,
provider=provider,
factory=factory,
work=work,
seed=job.seed or 0,
)
if live:
print(f"runtime validation: proved {len(live)} cases on the live runtime", flush=True)
except Exception as exc: # noqa: BLE001 - live proofs only add evidence
print(f"runtime validation: live proofs not run: {type(exc).__name__}: {exc}", flush=True)
if generic:
await provider.reset(runtime, work_directory=work)
reset_world = await factory.create(
Expand Down
11 changes: 8 additions & 3 deletions src/fi/alk/harness/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ def suite_server(
) -> ToolServer:
"""Whole-suite operations the single-scenario tools would take one call per scenario to do."""
from .catalogue import load_catalogue
from .proofs import record_proof
from .proofs import proof_key, record_proof
from .prove import prove
from .scenario_tools import write_scenarios

Expand Down Expand Up @@ -205,12 +205,17 @@ async def prove_scenarios(args: dict[str, Any]) -> dict[str, Any]:
lines.append(f"{name}: not in the suite")
continue
proof = prove(scenario, catalogue, out)
record_proof(out, scenario.scenario_key, proof)
record_proof(out, proof_key(scenario), proof, solution_steps=len(scenario.solution))
gates = ", ".join(f"{gate} {'yes' if held else 'no'}" for gate, held in proof.gates().items())
lines.append(
f"{name}: {'holds' if proof.holds else 'does not hold'} ({gates})"
+ ("" if proof.holds else f". {proof.why()}")
+ (" Judged-only: nothing observable to check." if proof.judged_only else "")
+ (
" Judged-only: nothing observable to check."
+ ("" if proof.controls_failed else f" {proof.why()}")
if proof.judged_only
else ""
)
)
return _ok("\n".join(lines) or "name at least one scenario")

Expand Down
61 changes: 56 additions & 5 deletions src/fi/alk/harness/bundle_author_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import tomllib
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any, Sequence
from typing import Any, Iterable, Sequence
from urllib.parse import urlsplit

import yaml
Expand Down Expand Up @@ -3226,7 +3226,24 @@ def exists_from(root: Path, path: str) -> bool:
return replace(plan, processes=tuple(processes))


def _copy_scenarios(authoring: Path, staging: Path, *, count: int) -> None:
def _folder_key(folder: Path) -> str:
try:
body = json.loads((folder / "scenario.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
return folder.name
return str(body.get("scenario_key") or body.get("name") or folder.name) if isinstance(body, dict) else folder.name


def _copy_scenarios(
authoring: Path,
staging: Path,
*,
count: int,
skip: frozenset[str] = frozenset(),
sealed: Path | None = None,
) -> None:
"""Copy the authoring scenarios into `staging`. A folder whose key is in `skip` keeps the copy
already sealed in `sealed` (its last good version), or is left out when it was never sealed."""
source = authoring / "scenarios"
if not source.is_dir():
raise BundleAuthorError(f"scenario_artifacts_missing: {source}")
Expand All @@ -3237,7 +3254,19 @@ def _copy_scenarios(authoring: Path, staging: Path, *, count: int) -> None:
raise BundleAuthorError(
f"scenario_artifacts_insufficient: requested {count}, found {len(folders)}"
)
held = (
{_folder_key(path): path for path in (sealed / "scenarios").iterdir() if path.is_dir()}
if skip and sealed is not None and (sealed / "scenarios").is_dir()
else {}
)
# A held copy never takes a folder name a case being written now uses; it is left out instead.
written = {path.name for path in folders[:count] if _folder_key(path) not in skip}
for source_folder in folders[:count]:
key = _folder_key(source_folder)
if key in skip:
if key in held and held[key].name not in written:
shutil.copytree(held[key], target / held[key].name)
continue
shutil.copytree(source_folder, target / source_folder.name)
for folder in sorted(path for path in target.iterdir() if path.is_dir()):
document = folder / "scenario.json"
Expand Down Expand Up @@ -3887,16 +3916,38 @@ def author_bundle_v2(
raise


def reseal_scenarios(authoring: str | Path) -> str | None:
def scenario_seal_problems(authoring: str | Path) -> dict[str, str]:
"""Each scenario folder a run could not load, by key, with the first line of why. Loaded one
folder at a time, the way a run loads them, so one broken case does not hide the rest."""
from .scenario_source import _declared_tool_names, _deterministic_names, _load_one

root = Path(authoring)
folders = sorted(path for path in (root / "scenarios").iterdir() if path.is_dir()) if (root / "scenarios").is_dir() else []
if not folders:
return {}
settled_in_code = _deterministic_names(root)
declared_tools = _declared_tool_names(root)
problems: dict[str, str] = {}
for folder in folders:
try:
_load_one(folder, settled_in_code=settled_in_code, declared_tools=declared_tools, defer_execution=False)
except Exception as exc: # noqa: BLE001 - every reason a run would refuse this folder
problems[_folder_key(folder)] = (str(exc).strip().splitlines() or [type(exc).__name__])[0][:300]
return problems


def reseal_scenarios(authoring: str | Path, *, skip: Iterable[str] = ()) -> str | None:
"""Carry an edited scenario set into the sealed bundle and seal it again.

Runs replay the bundle, never the authoring folders, so a scenario added or rewritten after
the build would otherwise be unknown to every Run. Only the scenario-owned files move; the
runtime, seed and processes stay exactly what was built and validated. A generic harness's
certificate binds the bundle digest, so it is reissued for the new set with the edit recorded
as a limitation: those scenarios were proved against the authoring world, not the runtime.
Returns the new digest, or None when there is no sealed bundle to update.
Returns the new digest, or None when there is no sealed bundle to update. A scenario whose key
is in `skip` keeps its last sealed version (or stays out if it was never sealed).
"""
skip = frozenset(skip)
authoring_root = Path(authoring)
bundle = authoring_root / "environment-bundle"
if not (bundle / BUNDLE_V2_MANIFEST).is_file():
Expand All @@ -3909,7 +3960,7 @@ def reseal_scenarios(authoring: str | Path) -> str | None:
shutil.copytree(bundle, staging)
(staging / BUNDLE_V2_MANIFEST).unlink()
shutil.rmtree(staging / "scenarios", ignore_errors=True)
_copy_scenarios(authoring_root, staging, count=count)
_copy_scenarios(authoring_root, staging, count=count, skip=skip, sealed=bundle)
if (authoring_root / CATALOGUE).is_file():
shutil.copy2(authoring_root / CATALOGUE, staging / CATALOGUE)
compiled, processes, metadata = _carry_contract(authoring_root, staging, manifest)
Expand Down
Loading
Loading