diff --git a/CHANGELOG.md b/CHANGELOG.md index d7fefe428..bf49aeea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable release changes for Agent Learning Kit are tracked here. Post-rc.1 increments on the release branch. Each one adds gates on top of the 66 proved at rc.1; the rc.1 entry below is historical and unchanged. +- Runtime validation checks scenarios on one database lane per CPU by default, capped by the + sandbox's cgroup CPU quota, instead of a fixed 16; `ALK_VALIDATION_DB_LANES` still overrides it. - Docs corpus: the full `docs/` tree (quickstarts, per-track guides, framework pages, reference material, and the `docs/llms.txt` machine index) with machine-checkable page metadata, enforced by the new `docs_executability` diff --git a/src/fi/alk/harness/authoring_runtime_validation.py b/src/fi/alk/harness/authoring_runtime_validation.py index 695e075c9..59581e99f 100644 --- a/src/fi/alk/harness/authoring_runtime_validation.py +++ b/src/fi/alk/harness/authoring_runtime_validation.py @@ -158,6 +158,27 @@ def _write_runtime_evidence( store.write_action_certification(action_report) +def _database_lanes() -> int: + """One lane per CPU by default: each check starts a worker inside its time budget.""" + from .process_runtime import _cgroup_cpu_quota + + configured = os.environ.get("ALK_VALIDATION_DB_LANES") + if configured: + try: + return max(1, int(configured)) + except ValueError: + raise ValueError( + f"ALK_VALIDATION_DB_LANES must be a whole number, not {configured!r}" + ) from None + cpus = ( + len(os.sched_getaffinity(0)) + if hasattr(os, "sched_getaffinity") + else os.cpu_count() or 1 + ) + quota = _cgroup_cpu_quota() + return max(1, min(cpus, int(quota)) if quota else cpus) + + def _world_state_digest(world) -> str: """Return a value-level baseline fingerprint without persisting world contents.""" @@ -592,7 +613,7 @@ async def validate_once( requested_lanes = max(1, int(os.environ.get("ALK_VALIDATION_INSTANCES", "4") or 4)) # Setup checks touch only the world's database, so a single-database world checks each # scenario on its own clone of the sealed baseline instead of a whole agent stack. - database_lanes = max(1, int(os.environ.get("ALK_VALIDATION_DB_LANES", "16") or 16)) + database_lanes = _database_lanes() executor = ThreadPoolExecutor( max_workers=max(requested_lanes, database_lanes), thread_name_prefix="runtime-validation", diff --git a/tests/harness/test_authoring_runtime_validation.py b/tests/harness/test_authoring_runtime_validation.py index aa4e5ea04..dc3ea316a 100644 --- a/tests/harness/test_authoring_runtime_validation.py +++ b/tests/harness/test_authoring_runtime_validation.py @@ -5,6 +5,7 @@ import pytest from fi.alk.harness.authoring_runtime_validation import ( + _database_lanes, _fallback_diagnostic, _make_local_seed_files_readable, RuntimeValidationError, @@ -54,6 +55,25 @@ ) +def test_database_lanes_default_to_one_per_cpu(monkeypatch): + from fi.alk.harness import process_runtime + + monkeypatch.delenv("ALK_VALIDATION_DB_LANES", raising=False) + monkeypatch.setattr("os.sched_getaffinity", lambda _pid: {0, 1, 2}, raising=False) + monkeypatch.setattr("os.cpu_count", lambda: 64) + monkeypatch.setattr(process_runtime, "_cgroup_cpu_quota", lambda: None) + assert _database_lanes() == 3 + monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "") + assert _database_lanes() == 3 + monkeypatch.setattr(process_runtime, "_cgroup_cpu_quota", lambda: 2.0) + assert _database_lanes() == 2 + monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "16") + assert _database_lanes() == 16 + monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "abc") + with pytest.raises(ValueError, match="ALK_VALIDATION_DB_LANES"): + _database_lanes() + + def test_local_seed_handoff_opens_only_generated_seed_files(tmp_path): bundle = tmp_path / "bundle" seed = bundle / "seed" @@ -1003,6 +1023,150 @@ async def create(self, *_args, **_kwargs): assert calls == ["provision", "reset", "close"] +def test_database_clone_checks_use_the_lane_count(tmp_path, monkeypatch): + from fi.alk.harness import ( + action_certification, + authoring_runtime_validation, + bundle_author_v2, + certification, + contract, + hosted_entrypoint, + outbound, + process_preflight, + process_runtime, + scenario_source, + source_data_invariants, + tool_certification, + ) + + original = tmp_path / "execution-secrets.json" + original.write_text("{}") + authoring = tmp_path / "authoring" + (authoring / "generic-harness").mkdir(parents=True) + (authoring / "contract.json").write_text("{}") + clones = [] + + class Provider: + def __init__(self, **kwargs): + pass + + async def provision(self, *args, work_directory, **kwargs): + (work_directory / "artifacts").mkdir() + (work_directory / "artifacts" / "build.json").write_text("{}") + return [SimpleNamespace(endpoints={})] + + async def clone_world_database(self, runtime, name): + clones.append(name) + return runtime + + async def reset(self, *args, **kwargs): + pass + + async def close(self, **kwargs): + pass + + class World: + def read_only(self): + return self + + class Factory: + def __init__(self, work): + pass + + async def create(self, *args, **kwargs): + return World() + + class Store: + RUNTIME_EVIDENCE = CERTIFICATION = "unused" + + def __init__(self, root): + pass + + def read_source_model(self): + return SimpleNamespace(actions=[]) + + def write_action_certification(self, report): + pass + + async def no_invariants(*args, **kwargs): + return [] + + for module, name, value in ( + (authoring_runtime_validation, "_database_lanes", lambda: 3), + (authoring_runtime_validation, "_world_isolation_status", lambda output: None), + (authoring_runtime_validation, "_world_state_digest", lambda world: "baseline"), + ( + authoring_runtime_validation, + "_GeneratedWorldActionAdapter", + lambda world: None, + ), + ( + authoring_runtime_validation, + "_setup_check_keys", + lambda *args: [str(i) for i in range(10)], + ), + ( + authoring_runtime_validation, + "_write_runtime_evidence", + lambda **kwargs: None, + ), + (authoring_runtime_validation, "_freeze_certified_bundle", lambda *args: None), + (process_runtime, "ProcessRuntimeProvider", Provider), + (hosted_entrypoint, "ProcessWorldFactory", Factory), + (certification, "GenericHarnessArtifactStore", Store), + ( + contract, + "AgentContract", + SimpleNamespace(model_validate_json=lambda text: None), + ), + ( + tool_certification, + "certify_tool_inventory", + lambda *args, **kwargs: SimpleNamespace(tools=[]), + ), + ( + action_certification, + "certify_actions", + lambda *args, **kwargs: SimpleNamespace(actions=[]), + ), + (bundle_author_v2, "author_bundle_v2", lambda **kwargs: object()), + (process_preflight, "preflight_bundle", lambda *args, **kwargs: None), + ( + outbound, + "load_capabilities", + lambda *, unlink: SimpleNamespace(attempt_id="test", expires_at=None), + ), + (source_data_invariants, "author_invariants", no_invariants), + (source_data_invariants, "check_invariants", no_invariants), + ( + scenario_source, + "load_scenarios", + lambda bundle: [ + SimpleNamespace( + scenario_key=str(i), + setup=lambda world: None, + ready=lambda world: True, + ) + for i in range(10) + ], + ), + ): + monkeypatch.setattr(module, name, value) + job = SimpleNamespace( + agent=SimpleNamespace(secret_refs={}), + scenario_count=10, + seed=0, + runtime=SimpleNamespace(cpu_units=4, memory_mb=8192), + metadata={"generic_harness_v1": True}, + ) + + assert ( + asyncio.run(validate_once(job, tmp_path, authoring, secrets_path=original)) + == 10 + ) + assert sorted(set(clones)) == ["lane0", "lane1", "lane2"] + + def test_local_runtime_validation_does_not_require_hosted_capabilities( tmp_path, monkeypatch ):