From 25e7006d886aeb7b39284ac3fd7469e96ca2d1bc Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 10:59:17 +0530 Subject: [PATCH 1/5] fix(harness): check scenarios on one database lane per CPU so worker start-up fits the ready budget --- .../harness/authoring_runtime_validation.py | 18 +++++++++++++++++- .../test_authoring_runtime_validation.py | 9 +++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/authoring_runtime_validation.py b/src/fi/alk/harness/authoring_runtime_validation.py index 695e075c9..37f13661d 100644 --- a/src/fi/alk/harness/authoring_runtime_validation.py +++ b/src/fi/alk/harness/authoring_runtime_validation.py @@ -158,6 +158,22 @@ def _write_runtime_evidence( store.write_action_certification(action_report) +def _database_lanes() -> int: + """Scenarios checked at once on database clones: one per CPU unless the deployment sets it. + + Each check starts its own Python worker inside the phase budget, so lanes beyond the CPUs + only queue those start-ups until a ready check runs out of time before it begins. + """ + configured = os.environ.get("ALK_VALIDATION_DB_LANES") + if configured: + return max(1, int(configured)) + try: + cpus = len(os.sched_getaffinity(0)) + except AttributeError: + cpus = os.cpu_count() or 1 + return max(1, cpus) + + def _world_state_digest(world) -> str: """Return a value-level baseline fingerprint without persisting world contents.""" @@ -592,7 +608,7 @@ async def validate_once( requested_lanes = max(1, int(os.environ.get("ALK_VALIDATION_INSTANCES", "4") or 4)) # Setup checks touch only the world's database, so a single-database world checks each # scenario on its own clone of the sealed baseline instead of a whole agent stack. - database_lanes = max(1, int(os.environ.get("ALK_VALIDATION_DB_LANES", "16") or 16)) + database_lanes = _database_lanes() executor = ThreadPoolExecutor( max_workers=max(requested_lanes, database_lanes), thread_name_prefix="runtime-validation", diff --git a/tests/harness/test_authoring_runtime_validation.py b/tests/harness/test_authoring_runtime_validation.py index aa4e5ea04..3f34ec3fd 100644 --- a/tests/harness/test_authoring_runtime_validation.py +++ b/tests/harness/test_authoring_runtime_validation.py @@ -5,6 +5,7 @@ import pytest from fi.alk.harness.authoring_runtime_validation import ( + _database_lanes, _fallback_diagnostic, _make_local_seed_files_readable, RuntimeValidationError, @@ -54,6 +55,14 @@ ) +def test_database_lanes_default_to_one_per_cpu(monkeypatch): + monkeypatch.delenv("ALK_VALIDATION_DB_LANES", raising=False) + monkeypatch.setattr("os.sched_getaffinity", lambda _pid: {0, 1, 2, 3}, raising=False) + assert _database_lanes() == 4 + monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "16") + assert _database_lanes() == 16 + + def test_local_seed_handoff_opens_only_generated_seed_files(tmp_path): bundle = tmp_path / "bundle" seed = bundle / "seed" From 45cd1433b324b4be0e0c6374d7cc9520bed9e548 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 13:38:36 +0530 Subject: [PATCH 2/5] refactor(harness): shorten the database lane helper --- .../alk/harness/authoring_runtime_validation.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/fi/alk/harness/authoring_runtime_validation.py b/src/fi/alk/harness/authoring_runtime_validation.py index 37f13661d..e612de75d 100644 --- a/src/fi/alk/harness/authoring_runtime_validation.py +++ b/src/fi/alk/harness/authoring_runtime_validation.py @@ -159,19 +159,10 @@ def _write_runtime_evidence( def _database_lanes() -> int: - """Scenarios checked at once on database clones: one per CPU unless the deployment sets it. - - Each check starts its own Python worker inside the phase budget, so lanes beyond the CPUs - only queue those start-ups until a ready check runs out of time before it begins. - """ + """One lane per CPU by default: each check starts a worker inside its time budget.""" configured = os.environ.get("ALK_VALIDATION_DB_LANES") - if configured: - return max(1, int(configured)) - try: - cpus = len(os.sched_getaffinity(0)) - except AttributeError: - cpus = os.cpu_count() or 1 - return max(1, cpus) + cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() + return max(1, int(configured or cpus or 1)) def _world_state_digest(world) -> str: From 7011c01c28077d2324e6f1331ffefda376a717f2 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 14:00:09 +0530 Subject: [PATCH 3/5] fix(harness): cap validation lanes by the cgroup CPU quota and name a bad lane override --- CHANGELOG.md | 2 ++ .../harness/authoring_runtime_validation.py | 12 +++++++++-- .../test_authoring_runtime_validation.py | 20 +++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7fefe428..bf49aeea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable release changes for Agent Learning Kit are tracked here. Post-rc.1 increments on the release branch. Each one adds gates on top of the 66 proved at rc.1; the rc.1 entry below is historical and unchanged. +- Runtime validation checks scenarios on one database lane per CPU by default, capped by the + sandbox's cgroup CPU quota, instead of a fixed 16; `ALK_VALIDATION_DB_LANES` still overrides it. - Docs corpus: the full `docs/` tree (quickstarts, per-track guides, framework pages, reference material, and the `docs/llms.txt` machine index) with machine-checkable page metadata, enforced by the new `docs_executability` diff --git a/src/fi/alk/harness/authoring_runtime_validation.py b/src/fi/alk/harness/authoring_runtime_validation.py index e612de75d..198950b7a 100644 --- a/src/fi/alk/harness/authoring_runtime_validation.py +++ b/src/fi/alk/harness/authoring_runtime_validation.py @@ -160,9 +160,17 @@ def _write_runtime_evidence( def _database_lanes() -> int: """One lane per CPU by default: each check starts a worker inside its time budget.""" + from .process_runtime import _cgroup_cpu_quota + configured = os.environ.get("ALK_VALIDATION_DB_LANES") - cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() - return max(1, int(configured or cpus or 1)) + if configured: + try: + return max(1, int(configured)) + except ValueError: + raise ValueError(f"ALK_VALIDATION_DB_LANES must be a whole number, not {configured!r}") from None + cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() or 1 + quota = _cgroup_cpu_quota() + return max(1, min(cpus, int(quota)) if quota else cpus) def _world_state_digest(world) -> str: diff --git a/tests/harness/test_authoring_runtime_validation.py b/tests/harness/test_authoring_runtime_validation.py index 3f34ec3fd..2fad2b73d 100644 --- a/tests/harness/test_authoring_runtime_validation.py +++ b/tests/harness/test_authoring_runtime_validation.py @@ -56,11 +56,22 @@ def test_database_lanes_default_to_one_per_cpu(monkeypatch): + from fi.alk.harness import process_runtime + monkeypatch.delenv("ALK_VALIDATION_DB_LANES", raising=False) - monkeypatch.setattr("os.sched_getaffinity", lambda _pid: {0, 1, 2, 3}, raising=False) - assert _database_lanes() == 4 + monkeypatch.setattr("os.sched_getaffinity", lambda _pid: {0, 1, 2}, raising=False) + monkeypatch.setattr("os.cpu_count", lambda: 64) + monkeypatch.setattr(process_runtime, "_cgroup_cpu_quota", lambda: None) + assert _database_lanes() == 3 + monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "") + assert _database_lanes() == 3 + monkeypatch.setattr(process_runtime, "_cgroup_cpu_quota", lambda: 2.0) + assert _database_lanes() == 2 monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "16") assert _database_lanes() == 16 + monkeypatch.setenv("ALK_VALIDATION_DB_LANES", "abc") + with pytest.raises(ValueError, match="ALK_VALIDATION_DB_LANES"): + _database_lanes() def test_local_seed_handoff_opens_only_generated_seed_files(tmp_path): @@ -1006,10 +1017,15 @@ async def create(self, *_args, **_kwargs): runtime={"isolation": "dedicated_vm"}, ) + from fi.alk.harness import authoring_runtime_validation + + asked = [] + monkeypatch.setattr(authoring_runtime_validation, "_database_lanes", lambda: asked.append(1) or 3) assert ( asyncio.run(validate_once(job, tmp_path, authoring, secrets_path=original)) == 4 ) assert calls == ["provision", "reset", "close"] + assert asked def test_local_runtime_validation_does_not_require_hosted_capabilities( From 956e7770a16fe9a0e97904882d058d168d5d9835 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 15:45:12 +0530 Subject: [PATCH 4/5] test(harness): pin the database clone checks to the lane count --- .../test_authoring_runtime_validation.py | 114 +++++++++++++++++- 1 file changed, 109 insertions(+), 5 deletions(-) diff --git a/tests/harness/test_authoring_runtime_validation.py b/tests/harness/test_authoring_runtime_validation.py index 2fad2b73d..e3c67377b 100644 --- a/tests/harness/test_authoring_runtime_validation.py +++ b/tests/harness/test_authoring_runtime_validation.py @@ -1017,15 +1017,119 @@ async def create(self, *_args, **_kwargs): runtime={"isolation": "dedicated_vm"}, ) - from fi.alk.harness import authoring_runtime_validation - - asked = [] - monkeypatch.setattr(authoring_runtime_validation, "_database_lanes", lambda: asked.append(1) or 3) assert ( asyncio.run(validate_once(job, tmp_path, authoring, secrets_path=original)) == 4 ) assert calls == ["provision", "reset", "close"] - assert asked + + +def test_database_clone_checks_use_the_lane_count(tmp_path, monkeypatch): + from fi.alk.harness import ( + action_certification, + authoring_runtime_validation, + bundle_author_v2, + certification, + contract, + hosted_entrypoint, + outbound, + process_preflight, + process_runtime, + scenario_source, + source_data_invariants, + tool_certification, + ) + + original = tmp_path / "execution-secrets.json" + original.write_text("{}") + authoring = tmp_path / "authoring" + (authoring / "generic-harness").mkdir(parents=True) + (authoring / "contract.json").write_text("{}") + clones = [] + + class Provider: + def __init__(self, **kwargs): + pass + + async def provision(self, *args, work_directory, **kwargs): + (work_directory / "artifacts").mkdir() + (work_directory / "artifacts" / "build.json").write_text("{}") + return [SimpleNamespace(endpoints={})] + + async def clone_world_database(self, runtime, name): + clones.append(name) + return runtime + + async def reset(self, *args, **kwargs): + pass + + async def close(self, **kwargs): + pass + + class World: + def read_only(self): + return self + + class Factory: + def __init__(self, work): + pass + + async def create(self, *args, **kwargs): + return World() + + class Store: + RUNTIME_EVIDENCE = CERTIFICATION = "unused" + + def __init__(self, root): + pass + + def read_source_model(self): + return SimpleNamespace(actions=[]) + + def write_action_certification(self, report): + pass + + async def no_invariants(*args, **kwargs): + return [] + + for module, name, value in ( + (authoring_runtime_validation, "_database_lanes", lambda: 3), + (authoring_runtime_validation, "_world_isolation_status", lambda output: None), + (authoring_runtime_validation, "_world_state_digest", lambda world: "baseline"), + (authoring_runtime_validation, "_GeneratedWorldActionAdapter", lambda world: None), + (authoring_runtime_validation, "_setup_check_keys", lambda *args: [str(i) for i in range(10)]), + (authoring_runtime_validation, "_write_runtime_evidence", lambda **kwargs: None), + (authoring_runtime_validation, "_freeze_certified_bundle", lambda *args: None), + (process_runtime, "ProcessRuntimeProvider", Provider), + (hosted_entrypoint, "ProcessWorldFactory", Factory), + (certification, "GenericHarnessArtifactStore", Store), + (contract, "AgentContract", SimpleNamespace(model_validate_json=lambda text: None)), + (tool_certification, "certify_tool_inventory", lambda *args, **kwargs: SimpleNamespace(tools=[])), + (action_certification, "certify_actions", lambda *args, **kwargs: SimpleNamespace(actions=[])), + (bundle_author_v2, "author_bundle_v2", lambda **kwargs: object()), + (process_preflight, "preflight_bundle", lambda *args, **kwargs: None), + (outbound, "load_capabilities", lambda *, unlink: SimpleNamespace(attempt_id="test", expires_at=None)), + (source_data_invariants, "author_invariants", no_invariants), + (source_data_invariants, "check_invariants", no_invariants), + ( + scenario_source, + "load_scenarios", + lambda bundle: [ + SimpleNamespace(scenario_key=str(i), setup=lambda world: None, ready=lambda world: True) + for i in range(10) + ], + ), + ): + monkeypatch.setattr(module, name, value) + job = SimpleNamespace( + agent=SimpleNamespace(secret_refs={}), + scenario_count=10, + seed=0, + runtime=SimpleNamespace(cpu_units=4, memory_mb=8192), + metadata={"generic_harness_v1": True}, + ) + + assert asyncio.run(validate_once(job, tmp_path, authoring, secrets_path=original)) == 10 + assert sorted(set(clones)) == ["lane0", "lane1", "lane2"] def test_local_runtime_validation_does_not_require_hosted_capabilities( From ebfcff0cdd216db442473ab8223cb82c380b04b1 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 23 Oct 2026 15:55:38 +0530 Subject: [PATCH 5/5] style(harness): format the database lane changes --- .../harness/authoring_runtime_validation.py | 10 +++- .../test_authoring_runtime_validation.py | 53 +++++++++++++++---- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/src/fi/alk/harness/authoring_runtime_validation.py b/src/fi/alk/harness/authoring_runtime_validation.py index 198950b7a..59581e99f 100644 --- a/src/fi/alk/harness/authoring_runtime_validation.py +++ b/src/fi/alk/harness/authoring_runtime_validation.py @@ -167,8 +167,14 @@ def _database_lanes() -> int: try: return max(1, int(configured)) except ValueError: - raise ValueError(f"ALK_VALIDATION_DB_LANES must be a whole number, not {configured!r}") from None - cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() or 1 + raise ValueError( + f"ALK_VALIDATION_DB_LANES must be a whole number, not {configured!r}" + ) from None + cpus = ( + len(os.sched_getaffinity(0)) + if hasattr(os, "sched_getaffinity") + else os.cpu_count() or 1 + ) quota = _cgroup_cpu_quota() return max(1, min(cpus, int(quota)) if quota else cpus) diff --git a/tests/harness/test_authoring_runtime_validation.py b/tests/harness/test_authoring_runtime_validation.py index e3c67377b..dc3ea316a 100644 --- a/tests/harness/test_authoring_runtime_validation.py +++ b/tests/harness/test_authoring_runtime_validation.py @@ -1095,26 +1095,58 @@ async def no_invariants(*args, **kwargs): (authoring_runtime_validation, "_database_lanes", lambda: 3), (authoring_runtime_validation, "_world_isolation_status", lambda output: None), (authoring_runtime_validation, "_world_state_digest", lambda world: "baseline"), - (authoring_runtime_validation, "_GeneratedWorldActionAdapter", lambda world: None), - (authoring_runtime_validation, "_setup_check_keys", lambda *args: [str(i) for i in range(10)]), - (authoring_runtime_validation, "_write_runtime_evidence", lambda **kwargs: None), + ( + authoring_runtime_validation, + "_GeneratedWorldActionAdapter", + lambda world: None, + ), + ( + authoring_runtime_validation, + "_setup_check_keys", + lambda *args: [str(i) for i in range(10)], + ), + ( + authoring_runtime_validation, + "_write_runtime_evidence", + lambda **kwargs: None, + ), (authoring_runtime_validation, "_freeze_certified_bundle", lambda *args: None), (process_runtime, "ProcessRuntimeProvider", Provider), (hosted_entrypoint, "ProcessWorldFactory", Factory), (certification, "GenericHarnessArtifactStore", Store), - (contract, "AgentContract", SimpleNamespace(model_validate_json=lambda text: None)), - (tool_certification, "certify_tool_inventory", lambda *args, **kwargs: SimpleNamespace(tools=[])), - (action_certification, "certify_actions", lambda *args, **kwargs: SimpleNamespace(actions=[])), + ( + contract, + "AgentContract", + SimpleNamespace(model_validate_json=lambda text: None), + ), + ( + tool_certification, + "certify_tool_inventory", + lambda *args, **kwargs: SimpleNamespace(tools=[]), + ), + ( + action_certification, + "certify_actions", + lambda *args, **kwargs: SimpleNamespace(actions=[]), + ), (bundle_author_v2, "author_bundle_v2", lambda **kwargs: object()), (process_preflight, "preflight_bundle", lambda *args, **kwargs: None), - (outbound, "load_capabilities", lambda *, unlink: SimpleNamespace(attempt_id="test", expires_at=None)), + ( + outbound, + "load_capabilities", + lambda *, unlink: SimpleNamespace(attempt_id="test", expires_at=None), + ), (source_data_invariants, "author_invariants", no_invariants), (source_data_invariants, "check_invariants", no_invariants), ( scenario_source, "load_scenarios", lambda bundle: [ - SimpleNamespace(scenario_key=str(i), setup=lambda world: None, ready=lambda world: True) + SimpleNamespace( + scenario_key=str(i), + setup=lambda world: None, + ready=lambda world: True, + ) for i in range(10) ], ), @@ -1128,7 +1160,10 @@ async def no_invariants(*args, **kwargs): metadata={"generic_harness_v1": True}, ) - assert asyncio.run(validate_once(job, tmp_path, authoring, secrets_path=original)) == 10 + assert ( + asyncio.run(validate_once(job, tmp_path, authoring, secrets_path=original)) + == 10 + ) assert sorted(set(clones)) == ["lane0", "lane1", "lane2"]