Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ All notable release changes for Agent Learning Kit are tracked here.
Post-rc.1 increments on the release branch. Each one adds gates on top of the
66 proved at rc.1; the rc.1 entry below is historical and unchanged.

- Runtime validation checks scenarios on one database lane per CPU by default, capped by the
sandbox's cgroup CPU quota, instead of a fixed 16; `ALK_VALIDATION_DB_LANES` still overrides it.
- Docs corpus: the full `docs/` tree (quickstarts, per-track guides, framework
pages, reference material, and the `docs/llms.txt` machine index) with
machine-checkable page metadata, enforced by the new `docs_executability`
Expand Down
23 changes: 22 additions & 1 deletion src/fi/alk/harness/authoring_runtime_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,27 @@ def _write_runtime_evidence(
store.write_action_certification(action_report)


def _database_lanes() -> int:
"""One lane per CPU by default: each check starts a worker inside its time budget."""
from .process_runtime import _cgroup_cpu_quota

configured = os.environ.get("ALK_VALIDATION_DB_LANES")
if configured:
try:
return max(1, int(configured))
except ValueError:
raise ValueError(
f"ALK_VALIDATION_DB_LANES must be a whole number, not {configured!r}"
) from None
cpus = (
len(os.sched_getaffinity(0))
if hasattr(os, "sched_getaffinity")
else os.cpu_count() or 1
)
quota = _cgroup_cpu_quota()
return max(1, min(cpus, int(quota)) if quota else cpus)


def _world_state_digest(world) -> str:
"""Return a value-level baseline fingerprint without persisting world contents."""

Expand Down Expand Up @@ -592,7 +613,7 @@ async def validate_once(
requested_lanes = max(1, int(os.environ.get("ALK_VALIDATION_INSTANCES", "4") or 4))
# Setup checks touch only the world's database, so a single-database world checks each
# scenario on its own clone of the sealed baseline instead of a whole agent stack.
database_lanes = max(1, int(os.environ.get("ALK_VALIDATION_DB_LANES", "16") or 16))
database_lanes = _database_lanes()
executor = ThreadPoolExecutor(
max_workers=max(requested_lanes, database_lanes),
thread_name_prefix="runtime-validation",
Expand Down
164 changes: 164 additions & 0 deletions tests/harness/test_authoring_runtime_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import pytest

from fi.alk.harness.authoring_runtime_validation import (
_database_lanes,
_fallback_diagnostic,
_make_local_seed_files_readable,
RuntimeValidationError,
Expand Down Expand Up @@ -54,6 +55,25 @@
)


def test_database_lanes_default_to_one_per_cpu(monkeypatch):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low. This tests the helper, but nothing tests that validate_once uses it. I put line 602 back to the old fixed 16 and all 43 tests in this file still passed. Removing the cap at line 725 passed too. A validate_once test with a fake provider whose clone_world_database returns a value, _database_lanes patched to 3, and 10 scenarios, asserting 3 distinct lane* targets, would pin the fix itself.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added: the connect-only validate_once test now asserts it asks _database_lanes, and it fails if line 602 goes back to the fixed 16.

from fi.alk.harness import process_runtime

monkeypatch.delenv("ALK_VALIDATION_DB_LANES", raising=False)
monkeypatch.setattr("os.sched_getaffinity", lambda _pid: {0, 1, 2}, raising=False)
monkeypatch.setattr("os.cpu_count", lambda: 64)
monkeypatch.setattr(process_runtime, "_cgroup_cpu_quota", lambda: None)
assert _database_lanes() == 3
monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "")
assert _database_lanes() == 3
monkeypatch.setattr(process_runtime, "_cgroup_cpu_quota", lambda: 2.0)
assert _database_lanes() == 2
monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "16")
assert _database_lanes() == 16
monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "abc")
with pytest.raises(ValueError, match="ALK_VALIDATION_DB_LANES"):
_database_lanes()


def test_local_seed_handoff_opens_only_generated_seed_files(tmp_path):
bundle = tmp_path / "bundle"
seed = bundle / "seed"
Expand Down Expand Up @@ -1003,6 +1023,150 @@ async def create(self, *_args, **_kwargs):
assert calls == ["provision", "reset", "close"]


def test_database_clone_checks_use_the_lane_count(tmp_path, monkeypatch):
from fi.alk.harness import (
action_certification,
authoring_runtime_validation,
bundle_author_v2,
certification,
contract,
hosted_entrypoint,
outbound,
process_preflight,
process_runtime,
scenario_source,
source_data_invariants,
tool_certification,
)

original = tmp_path / "execution-secrets.json"
original.write_text("{}")
authoring = tmp_path / "authoring"
(authoring / "generic-harness").mkdir(parents=True)
(authoring / "contract.json").write_text("{}")
clones = []

class Provider:
def __init__(self, **kwargs):
pass

async def provision(self, *args, work_directory, **kwargs):
(work_directory / "artifacts").mkdir()
(work_directory / "artifacts" / "build.json").write_text("{}")
return [SimpleNamespace(endpoints={})]

async def clone_world_database(self, runtime, name):
clones.append(name)
return runtime

async def reset(self, *args, **kwargs):
pass

async def close(self, **kwargs):
pass

class World:
def read_only(self):
return self

class Factory:
def __init__(self, work):
pass

async def create(self, *args, **kwargs):
return World()

class Store:
RUNTIME_EVIDENCE = CERTIFICATION = "unused"

def __init__(self, root):
pass

def read_source_model(self):
return SimpleNamespace(actions=[])

def write_action_certification(self, report):
pass

async def no_invariants(*args, **kwargs):
return []

for module, name, value in (
(authoring_runtime_validation, "_database_lanes", lambda: 3),
(authoring_runtime_validation, "_world_isolation_status", lambda output: None),
(authoring_runtime_validation, "_world_state_digest", lambda world: "baseline"),
(
authoring_runtime_validation,
"_GeneratedWorldActionAdapter",
lambda world: None,
),
(
authoring_runtime_validation,
"_setup_check_keys",
lambda *args: [str(i) for i in range(10)],
),
(
authoring_runtime_validation,
"_write_runtime_evidence",
lambda **kwargs: None,
),
(authoring_runtime_validation, "_freeze_certified_bundle", lambda *args: None),
(process_runtime, "ProcessRuntimeProvider", Provider),
(hosted_entrypoint, "ProcessWorldFactory", Factory),
(certification, "GenericHarnessArtifactStore", Store),
(
contract,
"AgentContract",
SimpleNamespace(model_validate_json=lambda text: None),
),
(
tool_certification,
"certify_tool_inventory",
lambda *args, **kwargs: SimpleNamespace(tools=[]),
),
(
action_certification,
"certify_actions",
lambda *args, **kwargs: SimpleNamespace(actions=[]),
),
(bundle_author_v2, "author_bundle_v2", lambda **kwargs: object()),
(process_preflight, "preflight_bundle", lambda *args, **kwargs: None),
(
outbound,
"load_capabilities",
lambda *, unlink: SimpleNamespace(attempt_id="test", expires_at=None),
),
(source_data_invariants, "author_invariants", no_invariants),
(source_data_invariants, "check_invariants", no_invariants),
(
scenario_source,
"load_scenarios",
lambda bundle: [
SimpleNamespace(
scenario_key=str(i),
setup=lambda world: None,
ready=lambda world: True,
)
for i in range(10)
],
),
):
monkeypatch.setattr(module, name, value)
job = SimpleNamespace(
agent=SimpleNamespace(secret_refs={}),
scenario_count=10,
seed=0,
runtime=SimpleNamespace(cpu_units=4, memory_mb=8192),
metadata={"generic_harness_v1": True},
)

assert (
asyncio.run(validate_once(job, tmp_path, authoring, secrets_path=original))
== 10
)
assert sorted(set(clones)) == ["lane0", "lane1", "lane2"]


def test_local_runtime_validation_does_not_require_hosted_capabilities(
tmp_path, monkeypatch
):
Expand Down
Loading