From 896a48cd9b7e1e0899f70c6210779f57b78642b1 Mon Sep 17 00:00:00 2001 From: Repository verification Date: Sun, 20 Sep 2026 01:17:32 +0000 Subject: [PATCH] fix(improvement): enforce training provenance at held-out boundaries --- CHANGELOG.md | 6 + api-surface.json | 2 +- bench/CHANGELOG.md | 4 + bench/package.json | 2 +- docs/api/index.md | 4 +- docs/api/primitive-catalog.md | 2 +- docs/architecture.md | 5 +- docs/canonical-api.md | 15 +- docs/research/README.md | 1 + .../training-purpose-audit-2026-09-19.md | 255 ++++++++++++++++++ package.json | 2 +- scripts/verify-package-exports.mjs | 3 + src/improvement/candidate-validation.ts | 21 ++ src/improvement/method-execution.ts | 24 +- .../profile-improvement-harness.ts | 7 +- .../authored-profile-improvement.ts | 4 + src/intelligence/improvement-cycle.ts | 6 +- .../fixtures/agent-improvement-proposal.json | 10 +- .../agent-profile-improvement-proposal.json | 6 +- tests/authored-profile-improvement.test.ts | 51 ++++ tests/helpers/trained-profile.ts | 49 ++++ tests/improvement-cycle.test.ts | 96 +++++++ tests/profile-training-exposure.test.ts | 88 ++++++ tests/profile-training.test.ts | 22 +- 24 files changed, 653 insertions(+), 32 deletions(-) create mode 100644 docs/research/training-purpose-audit-2026-09-19.md create mode 100644 tests/helpers/trained-profile.ts create mode 100644 tests/profile-training-exposure.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index aeba09aaa..e0dcb39d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.244.0 + +**Migration:** `harness.train(...)` now requires its own `executionRef` for the trainer, serving adapter, and their dependencies. The evaluation identity bound to the harness is no longer substituted into the training receipt. The bound parent profile and validator are unchanged. + +Profile optimization and authored/optimized profile promotion refuse final-task content digests already present in either measured profile's declared training receipts, including ancestors and trainer-visible validation data. Baseline exposure is refused before optimizer construction or analyst work; selected-candidate exposure is refused before final measurement. Unrelated task namespaces are not conflated. This checks known exact content digests; it does not certify undeclared or semantically duplicated data as fresh. + ## 0.243.0 Training, optimization, and bound harnesses now share candidate-validator admission and invocation. **Migration:** validators must return `undefined` synchronously or throw. Promises and other return values are rejected instead of silently accepting an unchecked candidate; use a block body for side effects. Composed optimizer leaves obey the same rule, while original callbacks remain bound into execution identity. diff --git a/api-surface.json b/api-surface.json index dffaf00ad..b7a64f369 100644 --- a/api-surface.json +++ b/api-surface.json @@ -148,7 +148,7 @@ "PlannerError": "value 3b3330bf4164", "ProfileImprovementHarness": "type cde3e185f5f2", "ProfileImprovementHarnessRunOptions": "type 3d1ff1d78855", - "ProfileImprovementHarnessTrainOptions": "type f1982d2b20f7", + "ProfileImprovementHarnessTrainOptions": "type 3e00f5c6931c", "ProfileTrainer": "type 2907ac3022c3", "ProfileTrainerRequest": "type 542a10bc9b12", "RESEARCH_SUPERVISOR_SYSTEM_PROMPT": "value 373728f5643d", diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 6bc25daa1..1eee2d5ca 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.13.6 + +Follow Runtime 0.244.0 through the workspace dependency. Benchmark APIs and grading are unchanged. + ## 0.13.5 Consume Runtime 0.243.0 through the existing workspace dependency. Benchmark APIs and grading behavior are unchanged. diff --git a/bench/package.json b/bench/package.json index fdfcae30d..bc6f58141 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.13.5", + "version": "0.13.6", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/docs/api/index.md b/docs/api/index.md index 350260c5d..4a3dbf388 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -3599,6 +3599,8 @@ Exact execution identity bound at construction. > **train**(`options`): `Promise`\<[`ImproveTrainingResult`](#improvetrainingresult)\> +Train the bound profile with a separately pinned trainer and serving execution identity. + ###### Parameters ###### options @@ -7792,7 +7794,7 @@ Official SkillOpt configuration plus bounded Runtime findings context. ### ProfileImprovementHarnessTrainOptions -> **ProfileImprovementHarnessTrainOptions** = `Omit`\<[`ImproveTrainingOptions`](#improvetrainingoptions), `"mode"` \| `"executionRef"`\> +> **ProfileImprovementHarnessTrainOptions** = `Omit`\<[`ImproveTrainingOptions`](#improvetrainingoptions), `"mode"`\> *** diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index d1d6b2add..2427b0706 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.243.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.244.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/architecture.md b/docs/architecture.md index b46bb54a9..8970f2087 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -92,7 +92,10 @@ Parents and children resume together through Scope, including reconstructed nest Provider managers can recover their original invocation and reuse an accepted result. Recovery preserves unresolved work when the provider cannot prove its identity or completion. The file run lock coordinates one local owner; this does not implement distributed fencing. -**This recursive execution tree IS the product.** The three things we own +**This recursive execution tree is the execution substrate, not the success criterion.** +The product succeeds when retained learning improves future outcomes under the resource and +verification conditions in sections 0.5 and 9. More nodes or more machinery do not establish that. +The three things we own are small: (1) the **MCP** the agents share (`spawn · observe · steer · stop` + `define_check · run_check`); (2) the **profiles** (markdown — the only customization; "Drew" is one); (3) the **orchestrator** (`src/runtime/supervise/` — `Scope` + the diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 3b412a2ce..130a0d0be 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.243.0.** +> **Version 0.244.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.182.0 <0.183.0`. > `sandbox` must satisfy `>=0.36.4 <0.42.0`. @@ -285,6 +285,19 @@ Rule of thumb: `delegate` = "I don't care how"; `supervise` = "I authored the dr ### Checkpoint training is candidate construction, not promotion +The bound harness's `executionRef` identifies evaluation, not the trainer or serving adapter. +Supply a separate, explicit training `executionRef` to `harness.train(...)`; it is recorded +unchanged in the training receipt. The harness still binds the parent profile and validator. +Omitting the training identity is an admission failure, not a fallback to evaluation identity. + +Runtime's profile optimization and profile proposal paths reject final measurement when a +held-out scenario digest occurs in either measured profile's declared training receipts, +including ancestors and trainer-visible validation rows. Exporters should retain the same +canonical scenario content digest used by evaluation. This is an exact-digest check, not +proof against undeclared training, differently encoded duplicates, or semantic contamination. +Task names in different benchmark namespaces are not treated as interchangeable identities. +Diagnostic evaluation outside these promotion paths remains the evaluator's responsibility. + Training mode consumes a byte-pinned dataset envelope containing existing Eval export rows and the identities of every exposed training and validation task. Payloads are not rewritten. The trainer writes one bounded checkpoint; the serving port must independently verify its diff --git a/docs/research/README.md b/docs/research/README.md index 98f9f3f3e..2532a7f0d 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -18,6 +18,7 @@ The research files below are source history and focused design inputs, not compe | Doc | What it holds | |-----|---------------| +| [training-purpose-audit-2026-09-19.md](./training-purpose-audit-2026-09-19.md) | Training provenance defects, integration requirements, existing-owner reuse, and finite engineering/value completion criteria. | | [simplification-map-2026-09-07.md](./simplification-map-2026-09-07.md) | Twelve-reader map of the 0.196.0 tree: consumer-less subpaths, dead mechanisms, duplication, and the three files that hold the churn, with an ordered cut list. Raw findings in `.agent/simplification-map-2026-09-07/`. | | [learning-system-audit-2026-09-05.md](./learning-system-audit-2026-09-05.md) | Current-source audit of learning across Runtime, Eval, and Knowledge, with reproduced failures and a unification design. | | [rsi-atom-masterplan.md](./rsi-atom-masterplan.md) | Historical self-designing-atom plan. Distributed execution work is superseded by `agent-managed-compute/`. | diff --git a/docs/research/training-purpose-audit-2026-09-19.md b/docs/research/training-purpose-audit-2026-09-19.md new file mode 100644 index 000000000..55e3d4790 --- /dev/null +++ b/docs/research/training-purpose-audit-2026-09-19.md @@ -0,0 +1,255 @@ +# Training: purpose, critical audit, and completion criteria + +**Scope:** Runtime main `3505d81e7f85683bc25bbd46a6bcbfdaf9503a71`, including merged +#1287 and #1300; their training, profile optimization, proposal, release, and test paths. +This is a source-level self-audit with executable regressions, not an independent reviewer +approval or an organization-wide production certification. Runtime's [architecture](../architecture.md) +and [benchmark ownership/evidence ladder](../../bench/HARNESS.md) remain authoritative. + +## What are we actually building? + +The useful product is a system whose retained experience improves outcomes on future work, +not a trainer launcher, a recursive tree, or a collection of evidence types. A concrete +customer promise is: give Tangle a task distribution, an objective, and a resource envelope; +it executes, retains useful improvements, and demonstrates that subsequent work benefits. +The improvements may change instructions, tools, skills, retrieval, routing, organization, +or weights. A weight update has no special entitlement to be the winning mechanism. + +The existing ambition is already large: architecture section 0.5 includes complex software, +independently checked research, better learning methods, and transfer of how to learn. +Making that promise larger would not close the current gap. Make it discriminating: +separately establish a better specialist, a more efficient learning procedure, and transfer +to a new project or domain. Do not require all three before the first useful release, and +do not report evidence for one as evidence for the others. + +The architecture previously called both across-run improvement and the recursive execution +tree the product. That invites optimizing machinery rather than outcomes. The maintained +architecture now explicitly identifies the tree as the substrate and sections 0.5/9 as the +success criteria. This is a clarification of the existing ambition, not a new framework. + +## Findings and decisions + +### 1. The training receipt recorded the wrong execution identity — fixed here + +**Why is it here?** A receipt must identify the implementation and dependencies responsible +for a checkpoint and its serving proof. `ProfileImprovementHarness.train` instead overwrote +its caller's training identity with the harness's evaluation identity. Those identify +different operations; the bound evaluation callback does not execute the training job. +An old test asserted the substitution, so its green result protected a bad contract. + +**Decision:** require the existing training `executionRef` explicitly on `train`, preserve +it unchanged, and keep the bound parent/validator. No second identity system. A test checks +the receipt against the supplied training identity; another rejects omission before work. +The packed consumer also proves omission is a TypeScript error. This corrects an API +contract and therefore carries a minor release and migration note. + +### 2. Known training exposure was decorative at the final gate — fixed here + +**Why record task ancestry?** To prevent a measurement described as held out from reusing +known training inputs. The trainer checked its own train/validation partitions, but neither +Runtime's method materializer nor its authored/optimized profile proposal paths checked +training receipts against final measurement. Four authored-candidate regressions accepted +current or ancestor exposure in either arm; method regressions also reached evaluation. + +**Decision:** one internal check in `candidate-validation.ts`, called from existing +materialization and proposal boundaries. It checks both arms and all retained receipts, +including trainer-visible validation data. A contaminated baseline fails before method +construction or analyst work; a selected candidate fails before final measurement. + +This checks exact `contentDigest` against the existing evaluator's scenario digest. It does +not invent equivalence between a training benchmark name and an evaluator scenario kind, +and does not reveal held-out task identities through optimizer errors. Names coinciding +with different content remain admissible. Exporters must preserve the same canonical content +identity; differently encoded duplicates and undeclared training remain unproven. This is +not a general decontamination certificate and does not change standalone Eval APIs. + +### 3. The real checkpoint-to-worker path is not established — next integration gate + +**Why launch a trainer?** To produce an agent that subsequently performs useful work. +`tests/profile-training.test.ts` learns a scalar and uses an injected serving fixture. That +proves command execution, byte preservation, and receipt construction. It does not prove +an LLM checkpoint loads in the serving backend or is consumed by the real coding worker. + +The existing consumer item is [Blueprint #2473](https://github.com/tangle-network/blueprint-agent/issues/2473). +Its latest inspected comment identifies `feat/p5-trained-arm-reentry` and says coder-profile +binding, matrix dispatch, and publication refusal validation remain pending. Do not create +another campaign or call a second Runtime cleanup completion of that consumer item. + +**Done:** exported real trajectories reach one existing trainer, its real output reaches +one serving implementation, and the same production worker executes it in the existing +paired benchmark. Record the executed model, not just the requested route. A wrong-checkpoint +negative control must fail. Do not replace any of these joins with mocks for this claim. + +### 4. A serving evidence digest is not self-authenticating — adapter obligation + +**Why the serving port?** Runtime should not own every deployment backend. The injection +boundary is appropriate. However, `serve()` returns an artifact digest, model id, and evidence +digest; Runtime verifies consistency, not the existence or independence of the evidence. +A conforming fake port can pass. A hash identifies bytes, not who witnessed an event. + +**Decision:** retain the port. The real adapter must retain retrievable evidence of the +loaded artifact and an inference through the actual consuming path, tied to immutable +provider/route identity. A route string alone is not a deployment attestation. Do not add +another registry or signature scheme until the existing serving evidence path is exercised. + +### 5. One checkpoint file needs an explicit packaging convention — integration requirement + +**Why one file?** It makes atomic hashing and publication tractable, and can contain an +archive. It is not inherently wrong. But native model output may include weight shards, +adapter configuration, tokenizer/configuration, and a specific base model. Current Runtime +accepts one opaque file and carries forward the parent provider; it does not resolve these +requirements or prove every parent harness can use the new route. + +**Decision:** define and test one packaging convention in the adapter using existing +artifact/archive facilities. Pin its dependencies and base revision, and verify backend +compatibility. Do not invent a new tensor serialization or loosen byte checks to make a +single-file toy fixture look compatible with all backends. + +### 6. Training cost is absent from the result contract — blocks economic claims + +**Why account for cost?** A cheaper model can still lose after training, rollout generation, +rejected runs, validation, storage, and serving are included. `ProfileTrainer.execute` +returns a success/failure outcome with no cost settlement. The profile experiment's cost +ledger does not retroactively establish the cost of an earlier training job. + +**Decision:** the real training adapter must join the existing execution/usage and Eval +cost-ledger paths. Report unknown cost as unknown. Compare cost per independently accepted +task including amortized adaptation cost, with the amortization horizon explicit. This is +not justification for a second billing system, and a receipt-only demo cannot claim ROI. + +### 7. Remote effect uncertainty has no reconciliation operation here — production requirement + +**Why the uncertainty flags?** Returning `trainingMayExist`/`servingMayExist` is more honest +than pretending cancellation deleted a remote job. But a new invocation allocates a fresh +local directory; this module has no caller-owned idempotent resume/reconcile operation. +A repeated call can represent another training or serving effect. Process-group teardown +is not remote recovery, and an omitted deadline is not crash durability. + +**Decision:** bind a concrete managed adapter to the existing retained execution identity, +recovery, and terminal-outcome facilities. Test interrupted admission, response loss, and +cancellation acknowledgement. Do not implement another generic scheduler inside improvement. +No paid managed workflow should be advertised as retry-safe merely because local artifacts +survive. This PR does not claim to solve remote exactly-once execution. + +### 8. Embedded ancestry bounds conflict with indefinite training — Interface-owned design limit + +**Why retain ancestry?** Repeated training must not erase exposure history. Interface 2.10.0 +embeds the current receipt plus at most eight ancestors; each dataset inventory admits at +most 10,000 task identities. This is a bounded control-plane representation, not indefinite +continual learning. Merely deleting the limits would trade a product limit for unbounded +profile parsing, transport, hashing, and storage. + +**Decision:** when the real integration requires longer lineage, use content-addressed +receipt/inventory references with verified resolution in the portable contract owner. +Preserve missing-history refusal and the exposure check. Do not silently truncate, or make +Runtime maintain a competing lineage grammar. Prove a sequence beyond the old ancestry +boundary before claiming the limit has been removed. + +### 9. The dataset envelope is a bounded snapshot, not a universal data plane + +**Why is it here?** It ties opaque trainer payloads to exposed task identities without +reimplementing Eval's export policy. That is useful. The current implementation nevertheless +captures and parses one JSON document with a 128 MiB byte limit, and canonicalizes it for +validation. Large trajectory corpora need a different data transport, not larger buffers. +The `sft`/`dpo`/`grpo` tags do not implement those training algorithms. + +**Decision:** keep the bounded snapshot for the current contract. Exercise the existing +Eval exporter unchanged. Add a manifest/streaming adapter only when the real consumer +requires it, preserving exposure identities and byte digests; do not create another SFT +row normalizer in Runtime. Use upstream training implementations, not hand-written RL math. + +### 10. Failure diagnostics discard the command output — operational improvement + +**Why bound output?** Prevent runaway trainers from flooding the control process. The current +command runner counts stdout/stderr bytes but retains neither channel. A nonzero exit can +therefore explain very little. This is a genuine supportability cost, not a reason to return +unlimited logs in a public receipt. + +**Decision:** reuse bounded, private runtime artifacts for diagnostic streams, with explicit +retention and redaction. Keep receipt metadata and returned errors free of raw secrets or +training examples. Verify missing files, loader failures, termination, and byte-limit overflow +with useful diagnostics. Scope this to the real adapter path rather than adding an unrelated +logging abstraction while its deployment is still unproven. + +### 11. Validation types and operation names should explain their actual contract + +**Why validation?** Refuse candidates that cannot be materialized or compared legitimately. +The shared synchronous contract is coherent, but TypeScript's `void` return type permits +callbacks returning promises even though Runtime rejects those results. Documentation alone +cannot make compile-time and runtime acceptance identical. + +**Decision:** plan an explicitly typed acceptance contract at the owning API, with consumer +migration tests. Do not casually make all materializers asynchronous. Similarly, keep +training candidate construction distinct from a measured `ship` result. The existing direct +`improve(profile, trainingOptions)` avoids constructing an evaluation harness solely to +train; a new train-only wrapper is not justified by that boilerplate. + +### 12. Our own engineering process was rewarding infrastructure completion + +**Why tests and release metadata?** They prevent real regressions and broken packages. But +more tests, types, hashes, and generated documentation are not evidence that retained +learning helps. The earlier training-identity test even asserted the wrong behavior. + +**Decision:** every change needs either a demonstrated correctness failure, a measured +consumer simplification, or a discriminating product experiment. A new abstraction pays +for itself by deleting an existing implementation or closing a demonstrated integration +obstacle. Do not continue low-impact cleanup indefinitely while the real worker path is +unfinished. Preserve an independent review stage; bot auto-approval and author self-audit +must not be labelled completed independent reviews. + +## Existing software: what to use instead of rebuilding + +TRL already implements SFT, DPO, GRPO and other transformer-training methods, with PEFT and +vLLM integrations ([official documentation](https://huggingface.co/docs/trl/index), inspected +2026-09-19). vLLM already serves LoRA adapters and documents their base-model relationship +([official documentation](https://docs.vllm.ai/en/latest/features/lora/)). ART is another +upstream option for multi-step agent reinforcement training, not a reason to add a second +training loop in Runtime ([official repository](https://github.com/OpenPipe/ART)). These are +integration candidates, not claimed plug-and-play compatibility with Tangle. + +Use the trainer/backend already closest to the existing consumer's working stack, then +pin and test it. An exhaustive bake-off is not a prerequisite. Runtime's differentiated +work is faithful execution of the improved agent and safe adoption; Eval owns measurement, +Knowledge owns retained knowledge, and the consuming lab owns the learning experiment. + +## A finite definition of done + +**Engineering-complete:** one real export-to-trainer-to-serving-to-worker-to-evaluation-to- +activation/rollback path works through existing owners; declared identity is preserved; +known exposure cannot produce a held-out claim; supported interruption/retry/cancellation +outcomes are demonstrated; package consumers work without unpublished overrides. No +unresolved critical/high correctness finding remains in the supported path. Unsupported +backends and recovery modes must be explicit, not silently generalized from this one path. + +**Value-established:** the chosen learning procedure beats a preregistered strong baseline +on fresh work by a practically meaningful amount, with independent verification, comparable +resource accounting, uncertainty, failures, and human intervention reported. Compare retained +learning against the same system with learning disabled/reset and against the best existing +non-weight lever relevant to the task. Do not select a weak baseline to manufacture a win. +Weight training wins only if its improvement justifies its full costs for the target workload. + +**Continual-learning evidence:** repeat adaptation on new batches/projects, preserve history, +and compare retained versus reset state. A few successive runs demonstrate plumbing, not +statistical generalization. Determine sample size, project clustering, minimum useful effect, +and stopping rules before final measurement. Report specialist improvement, learning-efficiency +improvement, and cross-domain transfer separately. + +**Experiment-complete:** accepted, rejected under the tested conditions, or inconclusive with +the missing evidence and budget stated. A negative result finishes an experiment. It does +not require another abstraction, and an inconclusive result does not justify endless reruns. +Global research is not assigned a fictional finish date; each delivery has the above boundary. + +## The review loop and its stopping rule + +For each proposed change, ask: whose decision changes, which failure/benefit is observable, +what existing owner already handles it, what the smallest decisive test is, and what would +make us delete it. Reproduce the failure, fix the owner, run regression and packaged-consumer +checks, and review the modified behavior again. This round found wrong execution identity +and unused exposure provenance; the second pass extended the guard to both final-proposal +paths and added positive controls so training-only data is not mistakenly forbidden. + +Stop this code tranche after these defects, migrations, and independent review are resolved. +Continue the existing #2473 integration rather than starting another generic cleanup campaign. +Stop the product experiment at its declared decision boundary. The moonshot is a system +that learns to produce better verified outcomes, not a system that can generate infinite +work for its maintainers. diff --git a/package.json b/package.json index a25167109..3569f5095 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.243.0", + "version": "0.244.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 1fa2ef105..90b30ecc0 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -305,6 +305,9 @@ try { agent: async () => { throw new Error('type-only fixture') }, }) const harnessTraining: Promise = profileHarness.train(trainingOptions) + const { executionRef: _trainingExecution, ...missingTrainingExecution } = trainingOptions + // @ts-expect-error Evaluation identity cannot substitute for the trainer/serving identity. + profileHarness.train(missingTrainingExecution) if (trainingResult.succeeded) { const trainingReceipt: AgentTrainingReceipt = trainingResult.receipt const { timeoutMs: _timeout, ...withoutDeadline } = trainingOptions diff --git a/src/improvement/candidate-validation.ts b/src/improvement/candidate-validation.ts index ec055c601..2986155d9 100644 --- a/src/improvement/candidate-validation.ts +++ b/src/improvement/candidate-validation.ts @@ -1,5 +1,7 @@ +import type { Sha256Digest } from '@tangle-network/agent-interface' import { ConfigError } from '../errors' import type { ImproveCandidateValidationInput, ImproveCandidateValidator } from './improve-types' +import type { ReadonlyAgentProfile } from './profile-types' /** Validate the callback itself before any optimizer, trainer, or candidate work starts. */ export function assertCandidateValidator( @@ -23,3 +25,22 @@ export function validateProfileCandidate( throw new ConfigError('candidate validators must return void synchronously or throw') } } + +/** + * Refuse exact held-out content digests already declared in this profile's training lineage. + * Both training and trainer-visible validation rows are exposure. This is an exact + * provenance check, not fuzzy decontamination or proof that undeclared training was fresh. + */ +export function assertProfileTrainingIsHeldOut( + profile: ReadonlyAgentProfile, + heldOutDigests: ReadonlySet, +): void { + const training = profile.metadata?.training + if (!training) return + for (const receipt of [training.receipt, ...training.ancestors]) { + if (receipt.dataset.tasks.some((task) => heldOutDigests.has(task.contentDigest))) { + // Do not disclose held-out task identities or payloads to an optimizer via errors. + throw new ConfigError('known training exposure overlaps held-out evaluation') + } + } +} diff --git a/src/improvement/method-execution.ts b/src/improvement/method-execution.ts index 7c3da67d6..1ffa383fa 100644 --- a/src/improvement/method-execution.ts +++ b/src/improvement/method-execution.ts @@ -24,7 +24,11 @@ import { } from '@tangle-network/agent-interface' import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' import { ConfigError } from '../errors' -import { assertCandidateValidator, validateProfileCandidate } from './candidate-validation' +import { + assertCandidateValidator, + assertProfileTrainingIsHeldOut, + validateProfileCandidate, +} from './candidate-validation' import { copyImproveCost } from './improve-result' import type { ImproveCandidateValidationInput, @@ -364,15 +368,8 @@ export async function runMethodImprovement task.scenarioDigest)) const validatedCandidates = new Set() const materializeProfile = ( candidateSurface: Parameters[0], @@ -380,6 +377,7 @@ export async function runMethodImprovement, diff --git a/src/improvement/profile-improvement-harness.ts b/src/improvement/profile-improvement-harness.ts index ff0fbd68e..962add479 100644 --- a/src/improvement/profile-improvement-harness.ts +++ b/src/improvement/profile-improvement-harness.ts @@ -17,10 +17,7 @@ import type { import type { ReadonlyAgentProfile } from './profile-types' import type { ImproveTrainingOptions, ImproveTrainingResult } from './training' -export type ProfileImprovementHarnessTrainOptions = Omit< - ImproveTrainingOptions, - 'mode' | 'executionRef' -> +export type ProfileImprovementHarnessTrainOptions = Omit export interface CreateProfileImprovementHarnessOptions { /** Exact baseline profile. It is parsed, detached, and frozen at construction. */ @@ -60,6 +57,7 @@ export interface ProfileImprovementHarness run( options: ProfileImprovementHarnessRunOptions, @@ -108,7 +106,6 @@ export function createProfileImprovementHarness task.scenario.digest)) + assertProfileTrainingIsHeldOut(baselineProfile, heldOutDigests) + assertProfileTrainingIsHeldOut(candidateProfile, heldOutDigests) assertDirectCandidateReleaseWorkIsFresh(benchmark, candidateLineage, options.developmentScenarios) const experiment = sealAgentProfileImprovementExperiment({ kind: 'agent-profile-improvement-experiment', diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index baa01d218..e9cba54ce 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -89,6 +89,7 @@ import { verifiedResourceTextByDigest, verifyAgentCandidateBundle, } from '../candidate-execution/verify' +import { assertProfileTrainingIsHeldOut } from '../improvement/candidate-validation' import { rethrowAfterCleanup } from '../improvement/cleanup' import { type ImproveMethodOptions, @@ -693,6 +694,9 @@ export async function proposeAgentProfileImprovement task.scenario.digest)) + assertProfileTrainingIsHeldOut(profile, heldOutDigests) if ( options.improvement.costCeiling !== undefined && !numbersApproximatelyEqual(options.improvement.costCeiling, options.budgetUsd) @@ -741,7 +745,7 @@ export async function proposeAgentProfileImprovement { ) }) }) + +describe('trained profile release admission', () => { + for (const arm of ['baseline', 'candidate'] as const) { + for (const inherited of [false, true]) { + it(`refuses ${arm} training exposure (${inherited ? 'ancestor' : 'current'}) before measurement`, async () => { + const { options, observed, heldOutScenario } = setup() + const key = arm === 'baseline' ? 'profile' : 'candidateProfile' + let profile = trainedProfile(options[key], [ + { + benchmark: 'independently-named-training-export', + task: 'renamed-training-task', + contentDigest: heldOutScenario.scenarioDigest, + }, + ]) + if (inherited) + profile = trainedProfile(profile, [ + { + benchmark: 'fresh', + task: 'fresh', + contentDigest: canonicalCandidateDigest('fresh'), + }, + ]) + options[key] = profile + options.source.sourceDigest = options.stateDigest({ + identity: options.source.sourceIdentity, + profile: options.profile, + }) + options.candidateLineage.parentDigests = [options.source.sourceDigest] + await expect(proposeAuthoredAgentProfileImprovement(options)).rejects.toThrow( + /training exposure/, + ) + expect(observed).toHaveLength(0) + }) + } + } + it('allows fresh task content even when training task names coincide', async () => { + const { options, observed, heldOutScenario } = setup() + options.candidateProfile = trainedProfile(options.candidateProfile, [ + { + benchmark: heldOutScenario.kind, + task: heldOutScenario.id, + contentDigest: canonicalCandidateDigest('fresh'), + }, + ]) + const result = await proposeAuthoredAgentProfileImprovement(options) + expect(result.proposal.evaluation.decision.outcome).toBe('ship') + expect(observed).toHaveLength(minimumPairedRuns * 2) + }) +}) diff --git a/tests/helpers/trained-profile.ts b/tests/helpers/trained-profile.ts new file mode 100644 index 000000000..5989c34a8 --- /dev/null +++ b/tests/helpers/trained-profile.ts @@ -0,0 +1,49 @@ +import { + type AgentProfile, + type AgentTrainingTask, + agentTrainingTaskKey, + canonicalAgentProfileDigest, + canonicalCandidateDigest, + snapshotAgentProfile, + trainedModelIdForArtifact, +} from '@tangle-network/agent-interface' + +/** Schema-valid portable training evidence; no claim of real model training. */ +export function trainedProfile(profile: AgentProfile, tasks: AgentTrainingTask[]): AgentProfile { + const members = [...tasks].sort((a, b) => + agentTrainingTaskKey(a) < agentTrainingTaskKey(b) ? -1 : 1, + ) + const parent = profile.metadata?.training + const artifactDigest = canonicalCandidateDigest({ profile, members }) + const receipt = { + version: 1 as const, + dataset: { + digest: canonicalCandidateDigest({ members }), + taskSetDigest: canonicalCandidateDigest(members), + tasks: members, + }, + parentProfileDigest: canonicalAgentProfileDigest(profile), + parentReceiptDigest: parent ? canonicalCandidateDigest(parent.receipt) : null, + executionRef: canonicalCandidateDigest({ fixture: 'training-executor' }), + trainer: { + mode: 'managed' as const, + id: 'training-evidence-fixture', + revision: canonicalCandidateDigest({ fixture: 'trainer' }), + parameters: {}, + }, + checkpoint: { + artifactDigest, + artifactBytes: 1, + routerModelId: trainedModelIdForArtifact(artifactDigest), + servingDigest: canonicalCandidateDigest({ fixture: 'serving' }), + }, + } + return snapshotAgentProfile({ + ...profile, + model: { ...profile.model, default: receipt.checkpoint.routerModelId }, + metadata: { + ...profile.metadata, + training: { receipt, ancestors: parent ? [parent.receipt, ...parent.ancestors] : [] }, + }, + }) +} diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts index 49c607e6d..ec5a62b37 100644 --- a/tests/improvement-cycle.test.ts +++ b/tests/improvement-cycle.test.ts @@ -74,6 +74,8 @@ import { createProfileImprovementRunReceipt, } from './helpers/profile-improvement-fixture' +import { trainedProfile } from './helpers/trained-profile' + afterEach(() => { cleanupCandidateExperimentFixtures() cleanupCandidateFixtures() @@ -720,6 +722,100 @@ describe('agent improvement lifecycle', { timeout: 30_000 }, () => { expect(budgetMeasurements).toBe(0) }) + for (const arm of ['baseline', 'selected'] as const) { + it(`refuses ${arm} training exposure to the release suite before measurement`, async () => { + const template = createProfileImprovementFixture() + const { digest: _digest, ...task } = template.evaluation.experiment.benchmark.tasks[0]! + const { agent: optimize, executionRef, ...improvement } = improvementOptions() + const untrained: AgentProfile = { prompt: { systemPrompt: 'BASELINE' } } + const exposure = [{ benchmark: 'export', task: 'alias', contentDigest: task.scenario.digest }] + const profile = arm === 'baseline' ? trainedProfile(untrained, exposure) : untrained + const candidate = trainedProfile({ prompt: { systemPrompt: 'PROMOTED' } }, exposure) + const stateDigest = ({ profile: value }: { identity: string; profile: AgentProfile }) => + canonicalCandidateDigest(value) + let analysisCalls = 0 + let methodCalls = 0 + let measured = 0 + await expect( + proposeAgentProfileImprovement({ + runId: `trained-release-${arm}`, + budgetUsd: minimumPairedRuns * 2 + 1, + source: { + kind: 'platform-agent-profile', + sourceIdentity: 'profile-support', + sourceDigest: stateDigest({ identity: 'profile-support', profile }), + sourceRevision: 7, + }, + profile, + stateDigest, + analysis: { + registry: { + list: () => [{ id: 'improvement' }], + run: async () => { + analysisCalls++ + return { + run_id: 'training-exposure', + correlation_id: 'training-exposure', + started_at: '2026-07-27T00:00:00.000Z', + ended_at: '2026-07-27T00:00:01.000Z', + findings: [finding], + per_analyst: [], + total_cost_usd: 0, + total_cost_provenance: { kind: 'observed' as const, usd: 0 }, + } + }, + }, + inputs: {}, + findingsStore: null, + }, + improvement: { + ...improvement, + surface: 'agent-profile', + method: { + name: 'trained-candidate', + async optimize() { + methodCalls++ + return { + winnerSurface: JSON.stringify(candidate), + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, + } + }, + }, + }, + benchmark: { + tasks: [task], + reps: minimumPairedRuns, + seeds: Array.from({ length: minimumPairedRuns }, (_, i) => i + 51) as [ + number, + ...number[], + ], + policy: template.evaluation.experiment.policy, + }, + executor: { + executionRef: { + kind: 'agent-profile-improvement-execution-ref', + identity: 'test', + digest: executionRef, + }, + optimize, + measure: async (input) => { + measured++ + return createProfileImprovementRunReceipt(input, input.arm === 'candidate' ? 1 : 0) + }, + }, + }), + ).rejects.toThrow(/training exposure/) + expect(measured).toBe(0) + expect(analysisCalls).toBe(arm === 'baseline' ? 0 : 1) + expect(methodCalls).toBe(arm === 'baseline' ? 0 : 1) + }) + } + it('rejects a profile measurement task previously visible to the optimizer', async () => { const template = createProfileImprovementFixture() const task = template.evaluation.experiment.benchmark.tasks[0] diff --git a/tests/profile-training-exposure.test.ts b/tests/profile-training-exposure.test.ts new file mode 100644 index 000000000..15b8ab693 --- /dev/null +++ b/tests/profile-training-exposure.test.ts @@ -0,0 +1,88 @@ +import { campaignScenarioIdentity } from '@tangle-network/agent-eval/campaign' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { improve } from '../src/improvement/improve' +import { improvementFinding, improvementOptions } from './helpers/improvement-method-fixture' +import { trainedProfile } from './helpers/trained-profile' + +describe('training exposure in method evaluation', () => { + for (const inherited of [false, true]) { + it(`refuses exposed final tasks from ${inherited ? 'ancestor' : 'current'} training before method construction`, async () => { + const options = improvementOptions() + let factoryCalls = 0 + let agentCalls = 0 + let profile = trainedProfile({ prompt: { systemPrompt: 'BASELINE' } }, [ + { + benchmark: 'export', + task: 'alias', + contentDigest: campaignScenarioIdentity(options.testScenarios[0]!).scenarioDigest, + }, + ]) + if (inherited) + profile = trainedProfile(profile, [ + { benchmark: 'fresh', task: 'fresh', contentDigest: canonicalCandidateDigest('fresh') }, + ]) + await expect( + improve(profile, { + ...options, + findings: [improvementFinding], + method: (context) => { + factoryCalls++ + return options.method(context) + }, + agent: async (...args) => { + agentCalls++ + return options.agent(...args) + }, + }), + ).rejects.toThrow(/training exposure/) + expect(factoryCalls).toBe(0) + expect(agentCalls).toBe(0) + }) + } + + for (const exposedToFinal of [true, false]) { + it(`${exposedToFinal ? 'refuses' : 'accepts'} an optimizer-selected trained profile ${exposedToFinal ? 'exposed to final test content' : 'exposed only to development content'}`, async () => { + const options = improvementOptions() + const profile: AgentProfile = { prompt: { systemPrompt: 'BASELINE' } } + const trainingScenario = exposedToFinal + ? options.testScenarios[0]! + : options.trainScenarios[0]! + const candidate = trainedProfile({ prompt: { systemPrompt: 'PROMOTED' } }, [ + { + benchmark: 'export', + task: 'alias', + contentDigest: campaignScenarioIdentity(trainingScenario).scenarioDigest, + }, + ]) + let trainedAgentCalls = 0 + const result = improve(profile, { + ...options, + surface: 'agent-profile', + findings: [improvementFinding], + method: (context) => { + const method = options.method(context) + return { + ...method, + async optimize(input) { + return { ...(await method.optimize(input)), winnerSurface: JSON.stringify(candidate) } + }, + } + }, + agent: async (candidate, scenario, context) => { + if (candidate.metadata?.training) trainedAgentCalls++ + return options.agent(candidate, scenario, context) + }, + }) + if (exposedToFinal) { + await expect(result).rejects.toThrow(/training exposure/) + expect(trainedAgentCalls).toBe(0) + } else { + const measured = await result + expect(measured.decision).toBe('ship') + expect(trainedAgentCalls).toBeGreaterThan(0) + await measured.dispose() + } + }) + } +}) diff --git a/tests/profile-training.test.ts b/tests/profile-training.test.ts index a3ad945e6..ebaf945ce 100644 --- a/tests/profile-training.test.ts +++ b/tests/profile-training.test.ts @@ -329,7 +329,7 @@ process.stdin.on('end', () => { ) }) - it('uses the bound harness parent identity execution reference and validator', async () => { + it('binds the parent and validator without substituting the training execution identity', async () => { await withFixture(async (options) => { const original = parent() const validations: boolean[] = [] @@ -347,11 +347,29 @@ process.stdin.on('end', () => { const result = await harness.train(options) assert(result.succeeded, JSON.stringify(result)) assert.equal(result.receipt.parentProfileDigest, harness.profileDigest) - assert.equal(result.receipt.executionRef, harness.executionRef) + assert.equal(result.receipt.executionRef, options.executionRef) + assert.notEqual(result.receipt.executionRef, harness.executionRef) assert.deepEqual(validations, [true, false]) }) }) + it('does not fall back to the evaluation identity when training identity is missing', async () => { + await withFixture(async (options) => { + const harness = createProfileImprovementHarness({ + profile: parent(), + executionRef: sha256Utf8('evaluation-only'), + agent: async () => { + throw new Error('must not execute an evaluation') + }, + }) + const { executionRef: _trainingRef, ...missingIdentity } = options + const result = await harness.train(missingIdentity as never) + assert(!result.succeeded) + assert.equal(result.stage, 'admission') + await assertNoProfile(options.outputDirectory) + }) + }) + it('retains the complete ancestry when training a trained parent', async () => { await withFixture(async (options) => { const first = await improve(parent(), options)