diff --git a/CHANGELOG.md b/CHANGELOG.md index 827ea291..05427844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.242.0 + +`improve(profile, { mode: 'training', ... })` and the bound harness's `train` method now produce a checkpoint-backed candidate with an Interface training receipt. The command trainer pins its executable and inputs, supplies only an explicit public environment, and cancels its POSIX process group. Managed trainers and verified serving adapters use the same typed boundary; they own remote job cleanup. + +The dataset envelope preserves existing Eval row payloads, inventories every exposed train/validation task, and refuses cross-partition reuse. Runtime verifies checkpoint bytes, artifact-addressed serving identity, and complete receipt ancestry. It reuses the existing cancellation and durable-file primitives: pre-dispatch cancellation starts no adapter, candidate validation cannot silently change checkpoint bytes, and the receipt is durable before profile publication. A late cancellation does not retract a committed profile; failure paths report uncertainty about external jobs. + +Requires `@tangle-network/agent-interface` `^2.10.0`, including the exact packed cohort. Training constructs a candidate; evaluation and promotion remain with the existing held-out gates. No GPU scheduler, managed trainer implementation, or Router deployment adapter is added. + ## 0.241.1 Coordination public-address signals now end when their listener closes or setup fails, as well as on manager cancellation. Hosted ingress adapters can retire a route before its local port is reused, without changing authentication, execution policy or agent capabilities. diff --git a/api-surface.json b/api-surface.json index 4e9fff13..da8d5ba8 100644 --- a/api-surface.json +++ b/api-surface.json @@ -29,6 +29,7 @@ "CandidateGenerator": "type d2b99fd9843d", "ChatModelCandidate": "type c35e223e502c", "ChatModelValidation": "type fc44fbff36e7", + "CheckpointServingPort": "type 7bf8caed3052", "CircuitBreakerConfig": "type d7631a064800", "ConfigError": "value a04b1851d315", "ControlBudget": "type b0404d2cb0b4", @@ -36,6 +37,7 @@ "ControlEvalResult": "type c7a2d96ce0ac", "ControlRunResult": "type 4994c45c1afc", "ControlStep": "type 69d8ab196483", + "ControlledTrainingCommand": "type e787701331f6", "Conversation": "type da37c7083ab5", "ConversationDriveState": "type 7320e9a7f523", "ConversationJournal": "type a8842d8d1d54", @@ -92,6 +94,8 @@ "ImproveScenarioPartitions": "type 37a3508406b1", "ImproveSkillsOptions": "type c1f5a69faefc", "ImproveSurface": "type b711b683b151", + "ImproveTrainingOptions": "type 8bfc42270e58", + "ImproveTrainingResult": "type 81c28cf9d5ab", "ImprovementCandidate": "type 0c22a91c6396", "ImprovementCodeCandidate": "type 588fa6d3b2f5", "ImprovementMaterializedProfilePopulationCandidate": "type 4eee039de8cf", @@ -142,8 +146,11 @@ "PersonaConversationResult": "type d642750715ee", "PersonaDriver": "type 14b302d4c33f", "PlannerError": "value 3b3330bf4164", - "ProfileImprovementHarness": "type aa5280c89d45", + "ProfileImprovementHarness": "type cde3e185f5f2", "ProfileImprovementHarnessRunOptions": "type 3d1ff1d78855", + "ProfileImprovementHarnessTrainOptions": "type f1982d2b20f7", + "ProfileTrainer": "type 2907ac3022c3", + "ProfileTrainerRequest": "type 542a10bc9b12", "RESEARCH_SUPERVISOR_SYSTEM_PROMPT": "value 373728f5643d", "ROLLOUT_POLICY_EXTENSION": "value 4fa0cbedb455", "RawTraceDistillerOptions": "type a04b8b0d09e1", @@ -203,6 +210,8 @@ "SupervisedKnowledgeUpdateOptions": "type e20062b4394e", "SupervisedKnowledgeUpdateResult": "type ac482862a4ae", "SupervisedKnowledgeUpdater": "type 90a6812eba78", + "TrainingBoundaryResult": "type f977a49cfd72", + "TrainingDatasetDocument": "type 5a17f19e9620", "TurnOrder": "type 573c9a049337", "ValidationError": "value 44437c30eb82", "Verifier": "type c596e7edac69", @@ -222,6 +231,7 @@ "commandVerifier": "value 75c0a10aca5d", "composeRuntimeHooks": "value a0c8fbe0631b", "createAgentKnowledgeReadinessCheck": "value 86edaf4f4ca1", + "createCommandProfileTrainer": "value 36c8ccf5d298", "createConversationBackend": "value 9fe923adfb90", "createIterableBackend": "value 314afcb51e55", "createKnowledgeImprovementActivationExecutor": "value b6efdb7cc158", @@ -243,7 +253,7 @@ "formatSupervisedKnowledgeTask": "value bdcf6b28157d", "generateSpanId": "value 2f8329045cac", "getModels": "value 95cb7c012c48", - "improve": "value 1303859a2197", + "improve": "value 45f11e9feb52", "isDelegatedLoopMode": "value d0f2042750ec", "knowledgeReadinessDeliverable": "value f6f33b24a926", "loopEventToOtelSpan": "value 63bec8b09ae0", diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index eae764f1..0f13076a 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.13.4 + +Require Interface `^2.10.0` and consume Runtime 0.242.0 through the published dependency ranges, keeping benchmark consumers on the checkpoint-training receipt contract. +Benchmark execution and grading behavior are unchanged. + ## 0.13.3 Support Sandbox 0.41 and consume Runtime 0.233.1 through the published dependency ranges. diff --git a/bench/package.json b/bench/package.json index 771c362e..d521260f 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.13.3", + "version": "0.13.4", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/docs/api/index.md b/docs/api/index.md index 1dc5eaf9..c1953478 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -3595,6 +3595,20 @@ Exact execution identity bound at construction. #### Methods +##### train() + +> **train**(`options`): `Promise`\<[`ImproveTrainingResult`](#improvetrainingresult)\> + +###### Parameters + +###### options + +[`ProfileImprovementHarnessTrainOptions`](#profileimprovementharnesstrainoptions) + +###### Returns + +`Promise`\<[`ImproveTrainingResult`](#improvetrainingresult)\> + ##### run() > **run**(`options`): `Promise`\<[`ImproveMethodResult`](#improvemethodresult)\> @@ -3719,6 +3733,258 @@ Receipt attribution phase supplied alongside `costLedger`. *** +### TrainingDatasetDocument + +#### Properties + +##### version + +> **version**: `1` + +##### format + +> **format**: `"sft"` \| `"dpo"` \| `"grpo"` + +##### rows + +> **rows**: `object`[] + +Existing Eval export rows, without rewriting their payloads. Include every exposed partition. + +###### task + +> **task**: `AgentTrainingTask` + +###### partition + +> **partition**: `"train"` \| `"validation"` + +###### data + +> **data**: `unknown` + +*** + +### ProfileTrainerRequest + +#### Properties + +##### version + +> **version**: `1` + +##### invocationId + +> **invocationId**: `string` + +##### datasetPath + +> **datasetPath**: `string` + +##### checkpointPath + +> **checkpointPath**: `string` + +##### parentProfilePath + +> **parentProfilePath**: `string` + +##### parentProfileDigest + +> **parentProfileDigest**: `` `sha256:${string}` `` + +##### parameters + +> **parameters**: `Record`\<`string`, `string` \| `number` \| `boolean` \| `null`\> + +##### executionRef + +> **executionRef**: `` `sha256:${string}` `` + +*** + +### ProfileTrainer + +Managed adapters use this same port: cancel the job on abort and download one exact checkpoint file. + +#### Properties + +##### identity + +> **identity**: `Omit`\<`AgentTrainingReceipt`\[`"trainer"`\], `"parameters"`\> + +#### Methods + +##### execute() + +> **execute**(`request`, `signal`): `Promise`\<[`TrainingBoundaryResult`](#trainingboundaryresult)\<`void`\>\> + +###### Parameters + +###### request + +`Readonly`\<[`ProfileTrainerRequest`](#profiletrainerrequest)\> + +###### signal + +`AbortSignal` + +###### Returns + +`Promise`\<[`TrainingBoundaryResult`](#trainingboundaryresult)\<`void`\>\> + +*** + +### CheckpointServingPort + +#### Methods + +##### serve() + +> **serve**(`input`): `Promise`\<[`TrainingBoundaryResult`](#trainingboundaryresult)\<\{ `routerModelId`: `string`; `artifactDigest`: `` `sha256:${string}` ``; `evidenceDigest`: `` `sha256:${string}` ``; \}\>\> + +Verify the immutable Router route independently of the trainer's output. + +###### Parameters + +###### input + +###### artifactPath + +`string` + +###### artifactDigest + +`` `sha256:${string}` `` + +###### artifactBytes + +`number` + +###### routerModelId + +`string` + +###### signal + +`AbortSignal` + +###### Returns + +`Promise`\<[`TrainingBoundaryResult`](#trainingboundaryresult)\<\{ `routerModelId`: `string`; `artifactDigest`: `` `sha256:${string}` ``; `evidenceDigest`: `` `sha256:${string}` ``; \}\>\> + +*** + +### ImproveTrainingOptions + +#### Properties + +##### mode + +> **mode**: `"training"` + +##### trainer + +> **trainer**: [`ProfileTrainer`](#profiletrainer) + +##### dataset + +> **dataset**: `object` + +###### path + +> **path**: `string` + +###### digest + +> **digest**: `` `sha256:${string}` `` + +##### parameters + +> **parameters**: `Record`\<`string`, `string` \| `number` \| `boolean` \| `null`\> + +##### executionRef + +> **executionRef**: `` `sha256:${string}` `` + +Pins trainer, serving adapter and their private dependencies, just like the bound profile harness. + +##### serving + +> **serving**: [`CheckpointServingPort`](#checkpointservingport) + +##### outputDirectory + +> **outputDirectory**: `string` + +##### timeoutMs + +> **timeoutMs**: `number` + +##### maxCheckpointBytes + +> **maxCheckpointBytes**: `number` + +##### signal? + +> `optional` **signal?**: `AbortSignal` + +##### validateCandidate? + +> `optional` **validateCandidate?**: [`ImproveCandidateValidator`](#improvecandidatevalidator) + +*** + +### ControlledTrainingCommand + +#### Properties + +##### id + +> **id**: `string` + +##### executable + +> **executable**: `object` + +###### path + +> **path**: `string` + +###### digest + +> **digest**: `` `sha256:${string}` `` + +##### args + +> **args**: `string`[] + +##### inputs + +> **inputs**: `object`[] + +Script/config files used by the command, verified before and after execution. + +###### path + +> **path**: `string` + +###### digest + +> **digest**: `` `sha256:${string}` `` + +##### environment + +> **environment**: `Record`\<`string`, `string`\> + +Explicit public environment only. Ambient credentials are never inherited. + +##### maxOutputBytes + +> **maxOutputBytes**: `number` + +*** + ### CreateKnowledgeImprovementActivationExecutorOptions #### Extends @@ -7518,6 +7784,12 @@ Official SkillOpt configuration plus bounded Runtime findings context. *** +### ProfileImprovementHarnessTrainOptions + +> **ProfileImprovementHarnessTrainOptions** = `Omit`\<[`ImproveTrainingOptions`](#improvetrainingoptions), `"mode"` \| `"executionRef"`\> + +*** + ### ProfileImprovementHarnessRunOptions > **ProfileImprovementHarnessRunOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<[`ImproveMethodOptions`](#improvemethodoptions)\<`TScenario`, `TArtifact`\>, `"executionRef"` \| `"agent"` \| `"validateCandidate"`\> & `object` @@ -7562,6 +7834,74 @@ Complete immutable profile value used during measured execution. *** +### TrainingBoundaryResult + +> **TrainingBoundaryResult**\<`T`\> = \{ `succeeded`: `true`; `value`: `T`; \} \| \{ `succeeded`: `false`; `reason`: `string`; \} + +#### Type Parameters + +##### T + +`T` + +*** + +### ImproveTrainingResult + +> **ImproveTrainingResult** = \{ `mode`: `"training"`; `succeeded`: `true`; `profile`: [`ReadonlyAgentProfile`](#readonlyagentprofile); `profileDigest`: `Sha256Digest`; `receipt`: `AgentTrainingReceipt`; `artifactPath`: `string`; `receiptPath`: `string`; `profilePath`: `string`; \} \| \{ `mode`: `"training"`; `succeeded`: `false`; `stage`: `"admission"` \| `"dataset"` \| `"training"` \| `"checkpoint"` \| `"serving"` \| `"profile"` \| `"persistence"`; `reason`: `string`; `outputDirectory?`: `string`; `servingMayExist`: `boolean`; `trainingMayExist`: `boolean`; `cleanupError?`: `string`; \} + +#### Union Members + +##### Type Literal + +\{ `mode`: `"training"`; `succeeded`: `true`; `profile`: [`ReadonlyAgentProfile`](#readonlyagentprofile); `profileDigest`: `Sha256Digest`; `receipt`: `AgentTrainingReceipt`; `artifactPath`: `string`; `receiptPath`: `string`; `profilePath`: `string`; \} + +*** + +##### Type Literal + +\{ `mode`: `"training"`; `succeeded`: `false`; `stage`: `"admission"` \| `"dataset"` \| `"training"` \| `"checkpoint"` \| `"serving"` \| `"profile"` \| `"persistence"`; `reason`: `string`; `outputDirectory?`: `string`; `servingMayExist`: `boolean`; `trainingMayExist`: `boolean`; `cleanupError?`: `string`; \} + +###### mode + +> **mode**: `"training"` + +###### succeeded + +> **succeeded**: `false` + +###### stage + +> **stage**: `"admission"` \| `"dataset"` \| `"training"` \| `"checkpoint"` \| `"serving"` \| `"profile"` \| `"persistence"` + +###### reason + +> **reason**: `string` + +###### outputDirectory? + +> `optional` **outputDirectory?**: `string` + +Partial artifacts are retained for diagnosis; they are not a runnable profile. + +###### servingMayExist + +> **servingMayExist**: `boolean` + +A serving request began; an interrupted adapter may still own a deployment. + +###### trainingMayExist + +> **trainingMayExist**: `boolean` + +A timed-out managed adapter may still own a remote training job. + +###### cleanupError? + +> `optional` **cleanupError?**: `string` + +*** + ### KnowledgeReadinessCheckResult > **KnowledgeReadinessCheckResult** = `boolean` \| \{ `ready`: `boolean`; `summary?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; \} @@ -8654,6 +8994,26 @@ Build the starting instruction for a coder agent tasked with implementing a new #### Call Signature +> **improve**(`profile`, `opts`): `Promise`\<[`ImproveTrainingResult`](#improvetrainingresult)\> + +Train and serve a checkpoint without implying that it improved held-out quality. + +##### Parameters + +###### profile + +`AgentProfile` + +###### opts + +[`ImproveTrainingOptions`](#improvetrainingoptions) + +##### Returns + +`Promise`\<[`ImproveTrainingResult`](#improvetrainingresult)\> + +#### Call Signature + > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `opts`): `Promise`\<[`ImproveMethodResult`](#improvemethodresult)\> Optimize one exact profile surface with a complete method. @@ -8973,6 +9333,24 @@ Persist a detached policy under the profile extension without mutating the input *** +### createCommandProfileTrainer() + +> **createCommandProfileTrainer**(`input`): [`ProfileTrainer`](#profiletrainer) + +Execute one pinned command without a shell, in the runtime-owned job directory. POSIX only. + +#### Parameters + +##### input + +[`ControlledTrainingCommand`](#controlledtrainingcommand) + +#### Returns + +[`ProfileTrainer`](#profiletrainer) + +*** + ### createKnowledgeImprovementActivationExecutor() > **createKnowledgeImprovementActivationExecutor**(`options`): [`KnowledgeImprovementActivationExecutor`](#knowledgeimprovementactivationexecutor) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index dd7525b1..651bafb8 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.241.1` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.242.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 288 exports. +Import from `@tangle-network/agent-runtime` — 298 exports. | Symbol | Kind | Summary | |---|---|---| @@ -31,6 +31,7 @@ Import from `@tangle-network/agent-runtime` — 288 exports. | `commandVerifier` | function | A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other | | `composeRuntimeHooks` | function | Merge several {@link RuntimeHooks} into one. Falsy entries are dropped (so you can | | `createAgentKnowledgeReadinessCheck` | function | Build the default readiness check backed by `@tangle-network/agent-knowledge` validation and scoring. | +| `createCommandProfileTrainer` | function | Execute one pinned command without a shell, in the runtime-owned job directory. POSIX only. | | `createConversationBackend` | function | Adapt a multi-participant conversation into the standard execution backend contract. | | `createIterableBackend` | function | Wrap any custom async-iterable stream into a typed `AgentExecutionBackend`. | | `createKnowledgeImprovementActivationExecutor` | function | Apply or restore one local knowledge candidate through the shared activation contract. | @@ -52,7 +53,7 @@ Import from `@tangle-network/agent-runtime` — 288 exports. | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | | `generateSpanId` | function | Mint a fresh 16-hex-character OTLP span id. Exported so a producer that must know a span's id | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | -| `improve` | function | Optimize one exact profile surface with a complete method. | +| `improve` | function | Train and serve a checkpoint without implying that it improved held-out quality. | | `isDelegatedLoopMode` | function | Type guard — returns true when `value` is a valid `DelegatedLoopMode` string. | | `knowledgeReadinessDeliverable` | function | Build the completion check a supervised KB update uses to stop only when the KB is ready. | | `loopEventToOtelSpan` | function | Convert a LoopTraceEvent into an OtelSpan for export. | @@ -146,6 +147,7 @@ Import from `@tangle-network/agent-runtime` — 288 exports. | `OfficialOptimizerContextOptions` | interface | Runtime context appended to an official optimizer's own configuration. | | `OpenAIChatTool` | interface | OpenAI Chat Completions tool descriptor. The shape mirrors the | | `ProfileImprovementHarness` | interface | A small, reusable front door over `improve(profile, options)`. | +| `ProfileTrainer` | interface | Managed adapters use this same port: cancel the job on abort and download one exact checkpoint file. | | `RouterEnv` | interface | Env keys the router base URL is resolved from. | | `RunRecord` | interface | Mandatory paper-grade fields for a single evaluation run. Optional | | `RuntimeHooks` | interface | The observation seam attached to a running loop (never to the portable genome). | @@ -173,7 +175,7 @@ Import from `@tangle-network/agent-runtime` — 288 exports. | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProfileImprovementHarnessOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `HaltContext`, `HaltSignal`, `ImproveCodeBaseOptions`, `ImproveCodeResult`, `ImproveCustomCodeGeneratorOptions`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveRuntimeCodeGeneratorOptions`, `ImproveSkillsOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `RawTraceDistillerOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgenticGeneratorExecutorForWorktree`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImprovementProfileCandidatePopulation`, `ImprovementProfilePopulationCandidate`, `ImprovementProfilePopulationLineage`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `ProfileImprovementHarnessRunOptions`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `ChatModelCandidate`, `CheckpointServingPort`, `ControlBudget`, `ControlEvalResult`, `ControlledTrainingCommand`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProfileImprovementHarnessOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `HaltContext`, `HaltSignal`, `ImproveCodeBaseOptions`, `ImproveCodeResult`, `ImproveCustomCodeGeneratorOptions`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveRuntimeCodeGeneratorOptions`, `ImproveSkillsOptions`, `ImproveTrainingOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `ProfileTrainerRequest`, `RawTraceDistillerOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `TrainingDatasetDocument`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgenticGeneratorExecutorForWorktree`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImprovementProfileCandidatePopulation`, `ImprovementProfilePopulationCandidate`, `ImprovementProfilePopulationLineage`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `ImproveTrainingResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `ProfileImprovementHarnessRunOptions`, `ProfileImprovementHarnessTrainOptions`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TrainingBoundaryResult`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 9ab256b3..55c6b655 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,11 +4,11 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.241.1.** +> **Version 0.242.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.182.0 <0.183.0`. > `sandbox` must satisfy `>=0.36.4 <0.42.0`. -> Portable profile and tool-part types come from `@tangle-network/agent-interface` `^2.6.0`. +> Portable profile and tool-part types come from `@tangle-network/agent-interface` `^2.10.0`. > > **`./kernel` is the execution kernel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/kernel` lives there — the recursive atom (`Scope`/`Supervisor`), the executor registry, budget conservation, the finalizer seam, analyst wiring, and the round-synchronous loop. > @@ -193,6 +193,7 @@ A thrown parent check reports a validation error through the existing driver fai | Optimize text or named components with upstream GEPA | `officialGepa({ recipe, ... })`, passed as `improve(...).method` from root `.` | a local GEPA approximation, prompt mutation loop, or silent fallback when Python is unavailable | | Optimize one text surface with Microsoft SkillOpt | `officialSkillOpt({ trainer, optimizer, ... })`, passed as `improve(...).method` from root `.` | Runtime-owned SkillOpt search or a silent local fallback | | Improve one profile coordinate | `improve(profile, { surface, executionRef, method, trainScenarios, selectionScenarios, testScenarios, judges, agent, costCeiling })` from root `.`; `executionRef` binds saved work to executable behavior, `agent` receives the exact complete candidate profile, and the total-cost option limits the whole run | an implicit per-surface optimizer, a method that sees final-test cases, an unmeasured profile mutation, or separate optimizer and final-test spend limits | +| Train model weights and return a receipted candidate | `improve(profile, { mode: 'training', ... })` or `createProfileImprovementHarness(...).train(...)` from root `.`; use `createCommandProfileTrainer` for a pinned local command, or provide a managed trainer and verified serving port | a second optimizer, Runtime-owned GPU deployment, rewritten Eval transcripts, or treating a trained checkpoint as a promotion verdict | | Inspect observed optimizer package, model, usage, cost, and resumed-run evidence before proposing a change | `createOptimizationActivationReceipt(result)` from `/intelligence` | reconstructing optimizer evidence from logs or trusting caller-authored metadata | | Compare complete optimization methods directly | `compareOptimizationMethods(...)` from `agent-eval/campaign` | comparing one method's training score to another method's final score | | Improve repository code | `improve({ surface: 'code', code, scenarios, judge, agent, budget })` from root `.` | passing code through a text optimizer or managing candidate worktrees in product code | @@ -281,3 +282,19 @@ Rule of thumb: `delegate` = "I don't care how"; `supervise` = "I authored the dr | **researcher → engineer** (gather, then build) | `defineStrategy(name, body)`: both agents in one body via `ctx.shot()` + `ctx.critique()` | `src/runtime/strategy.ts:789` | | **implement → verify** (build, then a SEPARATE checker gates it: selector ≠ judge) | `verify(spec)` as the `shape` | `src/runtime/personify/combinators.ts:333` | | **N-judge panel** (fan judges out, merge verdicts) | `panel(spec)` as the `shape` | `src/runtime/personify/combinators.ts:273` | + +### Checkpoint training is candidate construction, not promotion + +Training mode consumes a byte-pinned dataset envelope containing existing Eval export rows +and the identities of every exposed training and validation task. Payloads are not rewritten. +The trainer writes one bounded checkpoint; the serving port must independently verify its +artifact-addressed Router model identity. Runtime retains complete receipt ancestry, rechecks +checkpoint bytes after serving and candidate validation, and durably publishes the receipt +before the runnable profile. Use the existing benchmark and held-out gates to assess that profile. + +The controlled command trainer runs without a shell or inherited credentials and cancels its +POSIX process group. Managed training and serving adapters own their remote jobs and cleanup; +inspect the failure stage and the training/serving uncertainty flags rather than assuming a +timeout removed external resources. A cancellation before adapter dispatch starts no job. +Once profile publication commits, later cancellation does not retract the committed result. +This is a local execution primitive, not a durable remote-job scheduler or a Router deployment API. diff --git a/package.json b/package.json index 63267923..c3a20456 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.241.1", + "version": "0.242.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -156,7 +156,7 @@ "packageManager": "pnpm@11.17.0", "peerDependencies": { "@tangle-network/agent-eval": ">=0.182.0 <0.183.0", - "@tangle-network/agent-interface": "^2.6.0", + "@tangle-network/agent-interface": "^2.10.0", "@tangle-network/sandbox": ">=0.36.4 <0.42.0" }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 35f9f98d..405bef79 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,8 +19,8 @@ catalogs: specifier: '>=0.182.0 <0.183.0' version: 0.182.0 '@tangle-network/agent-interface': - specifier: ^2.6.0 - version: 2.6.0 + specifier: ^2.10.0 + version: 2.10.0 '@tangle-network/agent-knowledge': specifier: ^17.0.2 version: 17.0.2 @@ -58,10 +58,10 @@ importers: version: 0.9.6(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 17.0.2(@tangle-network/agent-eval@0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0) + version: 17.0.2(@tangle-network/agent-eval@0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.10.0) '@tangle-network/agent-profile-materialize': specifier: 'catalog:' - version: 0.19.0(@tangle-network/agent-interface@2.6.0) + version: 0.19.0(@tangle-network/agent-interface@2.10.0) '@tangle-network/agent-trace-contract': specifier: 'catalog:' version: 1.0.2 @@ -83,7 +83,7 @@ importers: version: 0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': specifier: 'catalog:' - version: 2.6.0 + version: 2.10.0 '@tangle-network/sandbox': specifier: 'catalog:' version: 0.41.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4))(viem@2.54.6(typescript@6.0.3)(zod@4.5.4)) @@ -137,10 +137,10 @@ importers: version: 0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': specifier: 'catalog:' - version: 2.6.0 + version: 2.10.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 17.0.2(@tangle-network/agent-eval@0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0) + version: 17.0.2(@tangle-network/agent-eval@0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.10.0) '@tangle-network/agent-runtime': specifier: workspace:^ version: link:.. @@ -1302,11 +1302,8 @@ packages: engines: {node: '>=20.19.0'} hasBin: true - '@tangle-network/agent-interface@2.6.0': - resolution: {integrity: sha512-tEByATif9oM5EEQB94Fytjt1Jtd70KxbWudKpwMQYyqXw2dcVFh2JTkeE8rIQP2F8KDLj9n5VHFkwg/ikbr7dg==} - - '@tangle-network/agent-interface@2.8.0': - resolution: {integrity: sha512-Li55RKjjB6VvkW5Ryba0QQLhd07zhD0ENYpacAHqYscyifJSeoh0xqJ8ws93qQ2Btp46P05Q42W6ej6wqX8Vjg==} + '@tangle-network/agent-interface@2.10.0': + resolution: {integrity: sha512-JdEU6VPIVnjmUlp/Jmrps43JicRh3WpwSh4i7d6tMRyA2XLr78QfmfW5KInDrRlYKArOocL7SobtX5dOOaD5eg==} '@tangle-network/agent-knowledge@17.0.2': resolution: {integrity: sha512-xPXGeDT8e7ag4Fz/lzZLLaNE29sWi9qyCr33KFP9kQ8MBBS2nw9R82KejJFDkvaKNm3zxI7iF4bms13PfJH3LA==} @@ -3565,7 +3562,7 @@ snapshots: '@tangle-network/agent-core@0.9.6(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4))': dependencies: - '@tangle-network/agent-interface': 2.6.0 + '@tangle-network/agent-interface': 2.10.0 zod: 4.5.4 optionalDependencies: '@modelcontextprotocol/sdk': 1.30.0(supports-color@10.2.2)(zod@4.5.4) @@ -3575,7 +3572,7 @@ snapshots: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.5.4) '@hono/node-server': 2.1.1(hono@4.13.7) '@tangle-network/agent-core': 0.9.6(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) - '@tangle-network/agent-interface': 2.8.0 + '@tangle-network/agent-interface': 2.10.0 '@tangle-network/agent-trace-contract': 1.0.2 hono: 4.13.7 linear-sum-assignment: 1.0.9 @@ -3584,36 +3581,30 @@ snapshots: transitivePeerDependencies: - '@modelcontextprotocol/sdk' - '@tangle-network/agent-interface@2.6.0': - dependencies: - '@noble/hashes': 2.4.0 - spdx-expression-parse: 5.0.0 - zod: 4.5.4 - - '@tangle-network/agent-interface@2.8.0': + '@tangle-network/agent-interface@2.10.0': dependencies: '@noble/hashes': 2.4.0 spdx-expression-parse: 5.0.0 zod: 4.5.4 - '@tangle-network/agent-knowledge@17.0.2(@tangle-network/agent-eval@0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0)': + '@tangle-network/agent-knowledge@17.0.2(@tangle-network/agent-eval@0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.10.0)': dependencies: '@tangle-network/agent-eval': 0.182.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) - '@tangle-network/agent-interface': 2.6.0 + '@tangle-network/agent-interface': 2.10.0 '@types/proper-lockfile': 4.1.4 proper-lockfile: 4.1.2 zod: 4.5.4 - '@tangle-network/agent-profile-materialize@0.19.0(@tangle-network/agent-interface@2.6.0)': + '@tangle-network/agent-profile-materialize@0.19.0(@tangle-network/agent-interface@2.10.0)': dependencies: - '@tangle-network/agent-interface': 2.6.0 + '@tangle-network/agent-interface': 2.10.0 '@tangle-network/agent-trace-contract@1.0.2': {} '@tangle-network/sandbox@0.41.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4))(viem@2.54.6(typescript@6.0.3)(zod@4.5.4))': dependencies: '@tangle-network/agent-core': 0.9.6(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) - '@tangle-network/agent-interface': 2.8.0 + '@tangle-network/agent-interface': 2.10.0 zod: 4.5.4 optionalDependencies: '@modelcontextprotocol/sdk': 1.30.0(supports-color@10.2.2)(zod@4.5.4) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 6c499c64..2b2f9238 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -24,7 +24,7 @@ catalog: '@tangle-network/agent-core': '>=0.9.6 <0.10.0' '@types/node': 26.4.0 '@tangle-network/agent-eval': '>=0.182.0 <0.183.0' - '@tangle-network/agent-interface': ^2.6.0 + '@tangle-network/agent-interface': '^2.10.0' '@tangle-network/agent-knowledge': ^17.0.2 '@tangle-network/agent-profile-materialize': '>=0.19.0 <0.20.0' '@tangle-network/agent-trace-contract': ^1.0.2 diff --git a/release/cohort.json b/release/cohort.json index 88f4d783..89aa01c0 100644 --- a/release/cohort.json +++ b/release/cohort.json @@ -4,8 +4,8 @@ "agentInterface": { "name": "@tangle-network/agent-interface", "repository": "tangle-network/agent-sdk", - "version": "2.6.0", - "ref": "a0f27e06c84090bc1c136bb17ba4f9774d080014" + "version": "2.10.0", + "ref": "eaefdcd75656c085516c3c0ee44c17a208fa6f09" }, "agentEval": { "name": "@tangle-network/agent-eval", diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index d563de25..bf4df135 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -214,14 +214,22 @@ try { AgentImprovementActivationOutcome, AgentImprovementProposal, AgentProfile, + AgentTrainingReceipt, CandidateExecutionEvidence, SandboxSizePreset, Sha256Digest, } from '@tangle-network/agent-interface' - import type { - ImproveMethodResult, - ImprovementProfileCandidatePopulation, - ImprovementProfilePopulationCandidateSource, + import { + createCommandProfileTrainer, + createProfileImprovementHarness, + improve, + type ControlledTrainingCommand, + type ImproveTrainingOptions, + type ImproveTrainingResult, + type ProfileTrainer, + type ImproveMethodResult, + type ImprovementProfileCandidatePopulation, + type ImprovementProfilePopulationCandidateSource, } from '@tangle-network/agent-runtime' import { driverAgent, @@ -286,6 +294,24 @@ try { declare const profileStateResolver: AgentImprovementProfileStateResolver declare const profileEvaluation: AgentImprovementEvaluation declare const activeProfile: AgentProfile + declare const trainingOptions: ImproveTrainingOptions + declare const commandOptions: ControlledTrainingCommand + declare const trainingResult: ImproveTrainingResult + const training: Promise = improve(activeProfile, trainingOptions) + const commandTrainer: ProfileTrainer = createCommandProfileTrainer(commandOptions) + const profileHarness = createProfileImprovementHarness({ + profile: activeProfile, + executionRef: trainingOptions.executionRef, + agent: async () => { throw new Error('type-only fixture') }, + }) + const harnessTraining: Promise = profileHarness.train(trainingOptions) + if (trainingResult.succeeded) { + const trainingReceipt: AgentTrainingReceipt = trainingResult.receipt + void trainingReceipt + } + void training + void commandTrainer + void harnessTraining declare const improvementResult: ImproveMethodResult const candidatePopulation: ImprovementProfileCandidatePopulation = improvementResult.candidatePopulation @@ -683,7 +709,7 @@ try { '--eval', ` const runtime = await import('@tangle-network/agent-runtime') - for (const name of ['improve', 'officialGepa', 'officialSkillOpt']) { + for (const name of ['improve', 'officialGepa', 'officialSkillOpt', 'createCommandProfileTrainer', 'createProfileImprovementHarness']) { if (typeof runtime[name] !== 'function') throw new Error('missing improvement export ' + name) } for (const name of [ diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 0b1518ed..2dea7eb7 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -3,8 +3,8 @@ * surface. Runtime extracts and materializes the profile value; agent-eval owns * optimization, disjoint data partitions, final-test scoring, and uncertainty. * - * Code is the sole exception. It uses Runtime's isolated git worktrees because - * checkout ownership and cleanup cannot cross a generic optimizer boundary. + * Code owns isolated git worktrees. Training owns checkpoint execution and + * serving receipts; unlike optimization, it does not make a promotion decision. * * @stable */ @@ -22,6 +22,11 @@ import type { ImproveResult, } from './improve-types' import { runMethodImprovement } from './method-execution' +import { + type ImproveTrainingOptions, + type ImproveTrainingResult, + runProfileTraining, +} from './training' export type { ImproveCandidateValidationInput, @@ -64,7 +69,23 @@ export type { ImproveSkillsOptions, ImproveSurface, } from './improve-types' +export type { + CheckpointServingPort, + ControlledTrainingCommand, + ImproveTrainingOptions, + ImproveTrainingResult, + ProfileTrainer, + ProfileTrainerRequest, + TrainingBoundaryResult, + TrainingDatasetDocument, +} from './training' +export { createCommandProfileTrainer } from './training' +/** Train and serve a checkpoint without implying that it improved held-out quality. */ +export function improve( + profile: AgentProfile, + opts: ImproveTrainingOptions, +): Promise /** * Optimize one exact profile surface with a complete method. */ @@ -80,8 +101,8 @@ export function improve( ): Promise> export async function improve( profileOrCode: AgentProfile | ImproveCodeRunOptions, - opts?: ImproveMethodOptions, -): Promise> { + opts?: ImproveMethodOptions | ImproveTrainingOptions, +): Promise | ImproveTrainingResult> { if (opts === undefined) { const code = profileOrCode as ImproveCodeRunOptions if (code?.surface !== 'code') { @@ -89,6 +110,9 @@ export async function improve( } return runCodeImprovement(code) } + if ('mode' in opts && opts.mode === 'training') { + return runProfileTraining(profileOrCode as AgentProfile, opts) + } if ((opts as { surface?: string }).surface === 'code') { throw new ConfigError("improve(): code takes one argument: improve({ surface: 'code', ... })") } @@ -98,5 +122,8 @@ export async function improve( `improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`, ) } - return runMethodImprovement(immutableCandidateValue(parsedProfile.data), opts) + return runMethodImprovement( + immutableCandidateValue(parsedProfile.data), + opts as ImproveMethodOptions, + ) } diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 5a97b758..5e0ed546 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -2,8 +2,8 @@ * `@tangle-network/agent-runtime` improvement. * * The public entry point is `improve()`. Complete agent-eval methods optimize - * profile surfaces. Runtime owns only code candidates that mutate an isolated - * git worktree through a pluggable `CandidateGenerator`. + * profile surfaces. Runtime owns isolated code candidates and trainer execution + * that returns checkpoint receipts, not promotion decisions. */ export { @@ -25,6 +25,9 @@ export { toolBuildPrompt, } from './build-prompts' export { + type CheckpointServingPort, + type ControlledTrainingCommand, + createCommandProfileTrainer, type ImproveCandidateValidationInput, type ImproveCandidateValidator, type ImproveCodeBaseOptions, @@ -64,7 +67,13 @@ export { type ImproveScenarioPartitions, type ImproveSkillsOptions, type ImproveSurface, + type ImproveTrainingOptions, + type ImproveTrainingResult, improve, + type ProfileTrainer, + type ProfileTrainerRequest, + type TrainingBoundaryResult, + type TrainingDatasetDocument, } from './improve' export type { CandidateGenerator } from './improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './mcp-serve-verifier' @@ -86,6 +95,7 @@ export { createProfileImprovementHarness, type ProfileImprovementHarness, type ProfileImprovementHarnessRunOptions, + type ProfileImprovementHarnessTrainOptions, } from './profile-improvement-harness' export type { DeepReadonly, ReadonlyAgentProfile } from './profile-types' export { diff --git a/src/improvement/profile-improvement-harness.ts b/src/improvement/profile-improvement-harness.ts index 06dd8387..b93d4b40 100644 --- a/src/improvement/profile-improvement-harness.ts +++ b/src/improvement/profile-improvement-harness.ts @@ -15,6 +15,12 @@ import type { ImproveProfileAgent, } from './improve-types' import type { ReadonlyAgentProfile } from './profile-types' +import type { ImproveTrainingOptions, ImproveTrainingResult } from './training' + +export type ProfileImprovementHarnessTrainOptions = Omit< + ImproveTrainingOptions, + 'mode' | 'executionRef' +> export interface CreateProfileImprovementHarnessOptions { /** Exact baseline profile. It is parsed, detached, and frozen at construction. */ @@ -54,6 +60,7 @@ export interface ProfileImprovementHarness run( options: ProfileImprovementHarnessRunOptions, ): Promise @@ -96,6 +103,15 @@ export function createProfileImprovementHarness) { if ( runOptions.validateCandidate !== undefined && diff --git a/src/improvement/training.ts b/src/improvement/training.ts new file mode 100644 index 00000000..7127e4be --- /dev/null +++ b/src/improvement/training.ts @@ -0,0 +1,573 @@ +import { spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { constants } from 'node:fs' +import { chmod, mkdir, mkdtemp, open, realpath, rm, writeFile } from 'node:fs/promises' +import { isAbsolute, join } from 'node:path' +import { + type AgentProfile, + type AgentProfileTraining, + type AgentTrainingDatasetIdentity, + type AgentTrainingReceipt, + type AgentTrainingTask, + agentProfileEnvironmentSchema, + agentTrainingDatasetIdentitySchema, + agentTrainingParametersSchema, + agentTrainingReceiptSchema, + agentTrainingTaskKey, + agentTrainingTaskSchema, + canonicalAgentProfileDigest, + canonicalCandidateBytes, + type Sha256Digest, + sha256DigestSchema, + snapshotAgentProfile, + trainedModelIdForArtifact, +} from '@tangle-network/agent-interface' +import { + canonicalCandidateDigest, + immutableCandidateValue, + sha256Bytes, +} from '../candidate-execution/digest' +import { runAbortable } from '../runtime/supervise/abortable' +import { + publishExclusiveDurableFile, + syncDurableDirectory, +} from '../runtime/supervise/durable-file' +import type { ImproveCandidateValidator } from './improve-types' +import type { ReadonlyAgentProfile } from './profile-types' + +export interface TrainingDatasetDocument { + version: 1 + format: 'sft' | 'dpo' | 'grpo' + /** Existing Eval export rows, without rewriting their payloads. Include every exposed partition. */ + rows: Array<{ task: AgentTrainingTask; partition: 'train' | 'validation'; data: unknown }> +} + +export interface ProfileTrainerRequest { + version: 1 + invocationId: string + datasetPath: string + checkpointPath: string + parentProfilePath: string + parentProfileDigest: Sha256Digest + parameters: AgentTrainingReceipt['trainer']['parameters'] + executionRef: Sha256Digest +} + +export type TrainingBoundaryResult = + | { succeeded: true; value: T } + | { succeeded: false; reason: string } + +/** Managed adapters use this same port: cancel the job on abort and download one exact checkpoint file. */ +export interface ProfileTrainer { + identity: Omit + execute( + request: Readonly, + signal: AbortSignal, + ): Promise> +} + +export interface CheckpointServingPort { + /** Verify the immutable Router route independently of the trainer's output. */ + serve(input: { + artifactPath: string + artifactDigest: Sha256Digest + artifactBytes: number + routerModelId: string + signal: AbortSignal + }): Promise< + TrainingBoundaryResult<{ + routerModelId: string + artifactDigest: Sha256Digest + evidenceDigest: Sha256Digest + }> + > +} + +export interface ImproveTrainingOptions { + mode: 'training' + trainer: ProfileTrainer + dataset: { path: string; digest: Sha256Digest } + parameters: AgentTrainingReceipt['trainer']['parameters'] + /** Pins trainer, serving adapter and their private dependencies, just like the bound profile harness. */ + executionRef: Sha256Digest + serving: CheckpointServingPort + outputDirectory: string + timeoutMs: number + maxCheckpointBytes: number + signal?: AbortSignal + validateCandidate?: ImproveCandidateValidator +} + +export type ImproveTrainingResult = + | { + mode: 'training' + succeeded: true + profile: ReadonlyAgentProfile + profileDigest: Sha256Digest + receipt: AgentTrainingReceipt + artifactPath: string + receiptPath: string + profilePath: string + } + | { + mode: 'training' + succeeded: false + stage: + | 'admission' + | 'dataset' + | 'training' + | 'checkpoint' + | 'serving' + | 'profile' + | 'persistence' + reason: string + /** Partial artifacts are retained for diagnosis; they are not a runnable profile. */ + outputDirectory?: string + /** A serving request began; an interrupted adapter may still own a deployment. */ + servingMayExist: boolean + /** A timed-out managed adapter may still own a remote training job. */ + trainingMayExist: boolean + cleanupError?: string + } + +export interface ControlledTrainingCommand { + id: string + executable: { path: string; digest: Sha256Digest } + args: string[] + /** Script/config files used by the command, verified before and after execution. */ + inputs: Array<{ path: string; digest: Sha256Digest }> + /** Explicit public environment only. Ambient credentials are never inherited. */ + environment: Record + maxOutputBytes: number +} + +function positiveLimit(value: number, maximum: number, label: string): void { + if (!Number.isSafeInteger(value) || value <= 0 || value > maximum) + throw new Error(`invalid ${label}`) +} + +async function hashFile( + path: string, + maximum: number, + signal: AbortSignal, + capture = false, +): Promise<{ + digest: Sha256Digest + bytes: number + content?: Buffer +}> { + signal.throwIfAborted() + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + const before = await file.stat() + if (!before.isFile() || before.size <= 0 || before.size > maximum) + throw new Error('artifact must be a bounded nonempty regular file') + const hash = createHash('sha256') + const chunks: Buffer[] = [] + let bytes = 0 + for await (const chunk of file.createReadStream({ autoClose: false, signal })) { + bytes += chunk.length + if (bytes > maximum) throw new Error('artifact exceeded its byte limit') + hash.update(chunk) + if (capture) chunks.push(Buffer.from(chunk)) + } + const after = await file.stat() + if ( + before.size !== bytes || + after.size !== bytes || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) { + throw new Error('artifact changed while being hashed') + } + return { + digest: `sha256:${hash.digest('hex')}`, + bytes, + ...(capture ? { content: Buffer.concat(chunks) } : {}), + } + } finally { + await file.close() + } +} + +/** Execute one pinned command without a shell, in the runtime-owned job directory. POSIX only. */ +export function createCommandProfileTrainer(input: ControlledTrainingCommand): ProfileTrainer { + const command = immutableCandidateValue(input) + positiveLimit(command.maxOutputBytes, 16 * 1024 * 1024, 'trainer output limit') + if ( + !command.id || + command.id.trim() !== command.id || + !Array.isArray(command.args) || + !command.args.every((arg) => typeof arg === 'string' && !arg.includes('\0')) + ) + throw new Error('invalid trainer command') + for (const file of [command.executable, ...command.inputs]) { + if (!isAbsolute(file.path)) throw new Error('trainer files must use absolute paths') + sha256DigestSchema.parse(file.digest) + } + for (const [name, value] of Object.entries(command.environment)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || value.includes('\0')) + throw new Error('invalid trainer environment') + } + agentProfileEnvironmentSchema.parse( + Object.fromEntries( + Object.entries(command.environment).map(([key, value]) => [key, { kind: 'public', value }]), + ), + ) + const identity = immutableCandidateValue({ + mode: 'command' as const, + id: command.id, + revision: canonicalCandidateDigest(command), + }) + agentTrainingReceiptSchema.shape.trainer.parse({ ...identity, parameters: {} }) + return Object.freeze({ + identity, + async execute( + request: Readonly, + signal: AbortSignal, + ): Promise> { + try { + if (process.platform === 'win32') + throw new Error('controlled trainers require POSIX process-group cancellation') + const verifyInputs = async () => { + for (const file of [command.executable, ...command.inputs]) { + if ((await hashFile(file.path, 1024 * 1024 * 1024, signal)).digest !== file.digest) { + throw new Error('trainer executable or input digest mismatch') + } + } + } + await verifyInputs() + signal.throwIfAborted() + await new Promise((resolve, reject) => { + const child = spawn(command.executable.path, command.args, { + cwd: join(request.checkpointPath, '..'), + env: command.environment, + shell: false, + detached: true, + stdio: ['pipe', 'pipe', 'pipe'], + }) + let failure: Error | undefined + let outputBytes = 0 + const stop = () => { + if (!child.pid) return + try { + process.kill(-child.pid, 'SIGKILL') + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') failure ??= error as Error + } + } + const abort = () => { + failure ??= new Error('trainer cancelled') + stop() + } + signal.addEventListener('abort', abort, { once: true }) + if (signal.aborted) abort() + const count = (chunk: Buffer) => { + outputBytes += chunk.length + if (outputBytes > command.maxOutputBytes) { + failure ??= new Error('trainer output limit exceeded') + stop() + } + } + child.stdout.on('data', count) + child.stderr.on('data', count) + child.stdin.on('error', (error) => { + failure ??= error + stop() + }) + child.on('error', (error) => { + failure ??= error + }) + // A successful parent may not leave descendants mutating the checkpoint. + child.on('exit', stop) + child.on('close', (code, exitSignal) => { + signal.removeEventListener('abort', abort) + stop() + if (failure) reject(failure) + else if (code !== 0 || exitSignal !== null) + reject(new Error(`trainer exited unsuccessfully (${code ?? exitSignal})`)) + else resolve() + }) + child.stdin.end(Buffer.from(canonicalCandidateBytes(request))) + }) + await verifyInputs() + return { succeeded: true, value: undefined } + } catch (error) { + return { + succeeded: false, + reason: error instanceof Error ? error.message : 'trainer failed', + } + } + }, + }) +} + +function datasetIdentity(bytes: Uint8Array): AgentTrainingDatasetIdentity { + const dataset = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(bytes), + ) as TrainingDatasetDocument + canonicalCandidateBytes(dataset) + if ( + !dataset || + typeof dataset !== 'object' || + Object.keys(dataset).sort().join(',') !== 'format,rows,version' || + dataset.version !== 1 || + !['sft', 'dpo', 'grpo'].includes(dataset.format) || + !Array.isArray(dataset.rows) || + dataset.rows.length === 0 || + dataset.rows.length > 1_000_000 + ) + throw new Error('invalid training dataset') + const tasks = new Map() + const partitions = new Map() + let trainingRows = 0 + for (const row of dataset.rows) { + if ( + !row || + typeof row !== 'object' || + Object.keys(row).sort().join(',') !== 'data,partition,task' || + !['train', 'validation'].includes(row.partition) + ) + throw new Error('every dataset row needs its exposure identity and partition') + if (row.partition === 'train') trainingRows++ + const task = agentTrainingTaskSchema.parse(row.task) + for (const identity of [JSON.stringify([task.benchmark, task.task]), task.contentDigest]) { + const previous = partitions.get(identity) + if (previous !== undefined && previous !== row.partition) + throw new Error('training and validation task partitions intersect') + partitions.set(identity, row.partition) + } + tasks.set(agentTrainingTaskKey(task), task) + } + if (trainingRows === 0) throw new Error('dataset contains no training rows') + const members = [...tasks.entries()] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([, task]) => task) + return agentTrainingDatasetIdentitySchema.parse({ + digest: sha256Bytes(bytes), + taskSetDigest: canonicalCandidateDigest(members), + tasks: members, + }) +} + +/** Training materializes a candidate; it never emits a ship verdict or changes a live agent. */ +export async function runProfileTraining( + profile: AgentProfile, + options: ImproveTrainingOptions, +): Promise { + let stage: Extract['stage'] = 'admission' + let jobDirectory: string | undefined + const controller = new AbortController() + const inputSignal = options.signal + const outputDirectory = options.outputDirectory + const timeoutMs = options.timeoutMs + let servingMayExist = false + let trainingMayExist = false + const abort = () => controller.abort(inputSignal?.reason ?? new Error('training cancelled')) + let timer: ReturnType | undefined + try { + if (options.mode !== 'training') throw new Error('training mode is required') + const parent = snapshotAgentProfile(profile) + if (!parent.harness || !parent.model?.default?.trim() || !parent.model.provider?.trim()) { + throw new Error('training requires an explicit parent harness, provider and model') + } + const parentProfileDigest = canonicalAgentProfileDigest(parent) + const parentBytes = Buffer.from(JSON.stringify(parent), 'utf8') + sha256DigestSchema.parse(options.executionRef) + sha256DigestSchema.parse(options.dataset.digest) + positiveLimit(options.timeoutMs, 7 * 24 * 60 * 60 * 1000, 'training timeout') + positiveLimit(options.maxCheckpointBytes, Number.MAX_SAFE_INTEGER, 'checkpoint byte limit') + if ( + typeof options.trainer?.execute !== 'function' || + typeof options.serving?.serve !== 'function' + ) + throw new Error('trainer and verified serving ports are required') + const trainer = immutableCandidateValue(options.trainer.identity) + const parameters = immutableCandidateValue( + agentTrainingParametersSchema.parse(options.parameters), + ) + agentTrainingReceiptSchema.shape.trainer.parse({ ...trainer, parameters }) + if (options.validateCandidate !== undefined && typeof options.validateCandidate !== 'function') + throw new Error('invalid candidate validator') + const execute = options.trainer.execute.bind(options.trainer) + const serve = options.serving.serve.bind(options.serving) + const executionRef = options.executionRef + const expectedDatasetDigest = options.dataset.digest + const sourceDataset = options.dataset.path + const maxCheckpointBytes = options.maxCheckpointBytes + const validateCandidate = options.validateCandidate + const ancestry: AgentProfileTraining['ancestors'] = parent.metadata?.training + ? [parent.metadata.training.receipt, ...parent.metadata.training.ancestors] + : [] + if (ancestry.length > 8) throw new Error('training receipt ancestry limit exceeded') + const validate = (candidate: AgentProfile, isBaseline: boolean) => { + const result: unknown = validateCandidate?.({ + profile: candidate, + surface: 'agent-profile', + candidateSurface: JSON.stringify(candidate), + value: candidate, + isBaseline, + }) + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + throw new Error('candidate validators must return void synchronously or throw') + } + } + validate(parent, true) + inputSignal?.addEventListener('abort', abort, { once: true }) + if (inputSignal?.aborted) abort() + timer = setTimeout(() => controller.abort(new Error('training deadline exceeded')), timeoutMs) + const signal = controller.signal + signal.throwIfAborted() + await mkdir(outputDirectory, { recursive: true }) + const outputRoot = await realpath(outputDirectory) + jobDirectory = await mkdtemp(join(outputRoot, 'training-')) + await chmod(jobDirectory, 0o700) + syncDurableDirectory(outputRoot) + const datasetPath = join(jobDirectory, 'dataset.json') + const parentProfilePath = join(jobDirectory, 'parent-profile.json') + const artifactPath = join(jobDirectory, 'checkpoint.bin') + stage = 'dataset' + const source = await hashFile(sourceDataset, 128 * 1024 * 1024, signal, true) + if (source.digest !== expectedDatasetDigest) throw new Error('training dataset digest mismatch') + const bytes = source.content! + if (bytes.length !== source.bytes) throw new Error('training dataset snapshot is incomplete') + const dataset = datasetIdentity(bytes) + await writeFile(datasetPath, bytes, { flag: 'wx', mode: 0o400 }) + await writeFile(parentProfilePath, parentBytes, { flag: 'wx', mode: 0o400 }) + const request = immutableCandidateValue({ + version: 1 as const, + invocationId: jobDirectory, + datasetPath, + checkpointPath: artifactPath, + parentProfilePath, + parentProfileDigest, + parameters, + executionRef, + }) + stage = 'training' + const trained = await runAbortable( + () => { + trainingMayExist = true + return execute(request, signal) + }, + signal, + 'training cancelled', + ) + signal.throwIfAborted() + if (trained?.succeeded !== true) + throw new Error(trained?.reason ?? 'trainer did not report success') + trainingMayExist = false + stage = 'checkpoint' + if ( + (await hashFile(datasetPath, 128 * 1024 * 1024, signal)).digest !== dataset.digest || + (await hashFile(parentProfilePath, parentBytes.byteLength, signal)).digest !== + sha256Bytes(parentBytes) + ) { + throw new Error('trainer changed its pinned inputs') + } + const artifact = await hashFile(artifactPath, maxCheckpointBytes, signal) + await chmod(artifactPath, 0o400) + const checkpoint = await open(artifactPath, constants.O_RDONLY | constants.O_NOFOLLOW) + try { + await checkpoint.sync() + } finally { + await checkpoint.close() + } + stage = 'serving' + const served = await runAbortable( + () => { + servingMayExist = true + return serve({ + artifactPath, + artifactDigest: artifact.digest, + artifactBytes: artifact.bytes, + routerModelId: trainedModelIdForArtifact(artifact.digest), + signal, + }) + }, + signal, + 'checkpoint serving cancelled', + ) + signal.throwIfAborted() + if (served?.succeeded !== true) + throw new Error(served?.reason ?? 'checkpoint serving is unverified') + if (served.value.artifactDigest !== artifact.digest) + throw new Error('Router serving evidence names a different checkpoint') + if ((await hashFile(artifactPath, maxCheckpointBytes, signal)).digest !== artifact.digest) + throw new Error('checkpoint changed during serving') + const receipt = immutableCandidateValue( + agentTrainingReceiptSchema.parse({ + version: 1, + dataset, + parentProfileDigest, + parentReceiptDigest: ancestry[0] ? canonicalCandidateDigest(ancestry[0]) : null, + executionRef, + trainer: { ...trainer, parameters }, + checkpoint: { + artifactDigest: artifact.digest, + artifactBytes: artifact.bytes, + routerModelId: served.value.routerModelId, + servingDigest: served.value.evidenceDigest, + }, + }), + ) + stage = 'profile' + const candidate = snapshotAgentProfile({ + ...parent, + model: { ...parent.model, default: receipt.checkpoint.routerModelId }, + metadata: { ...parent.metadata, training: { receipt, ancestors: ancestry } }, + }) + validate(candidate, false) + // The validator is caller code too; publish only the bytes the receipt actually names. + if ((await hashFile(artifactPath, maxCheckpointBytes, signal)).digest !== artifact.digest) + throw new Error('checkpoint changed during candidate validation') + stage = 'persistence' + const receiptPath = join(jobDirectory, 'receipt.json') + const profilePath = join(jobDirectory, 'profile.json') + const profileDigest = canonicalAgentProfileDigest(candidate) + // Reuse the runtime's no-clobber, fsynced publication primitive. Once the + // profile is committed, a late cancellation must not retract an observed result. + for (const [path, value] of [ + [receiptPath, receipt], + [profilePath, candidate], + ] as const) { + signal.throwIfAborted() + if (!publishExclusiveDurableFile(path, JSON.stringify(value), { mode: 0o400 })) + throw new Error('training publication path already exists') + } + return { + mode: 'training', + succeeded: true, + profile: candidate, + profileDigest, + receipt, + artifactPath, + receiptPath, + profilePath, + } + } catch (error) { + let cleanupError: string | undefined + if (jobDirectory) { + try { + await rm(join(jobDirectory, 'profile.json'), { force: true }) + syncDurableDirectory(jobDirectory) + } catch (failure) { + cleanupError = failure instanceof Error ? failure.message : 'profile cleanup failed' + } + } + return { + mode: 'training', + succeeded: false, + stage, + reason: error instanceof Error ? error.message : 'training failed', + servingMayExist, + trainingMayExist, + ...(cleanupError ? { cleanupError } : {}), + ...(jobDirectory ? { outputDirectory: jobDirectory } : {}), + } + } finally { + if (timer) clearTimeout(timer) + inputSignal?.removeEventListener('abort', abort) + } +} diff --git a/src/index.ts b/src/index.ts index 0cbf4004..d1d69882 100644 --- a/src/index.ts +++ b/src/index.ts @@ -81,8 +81,8 @@ export { ValidationError, } from './errors' // ── Improvement (self-improvement surfaces) ────────────────────────── -// Complete agent-eval methods optimize profile fields. Runtime owns only -// isolated code/worktree candidate execution. +// Complete agent-eval methods optimize profile fields. Runtime owns isolated +// code/worktree candidates and checkpoint-producing trainer execution. export { type AgenticGeneratorExecutorForWorktree, type AgenticGeneratorOptions, @@ -102,6 +102,9 @@ export { toolBuildPrompt, } from './improvement/build-prompts' export { + type CheckpointServingPort, + type ControlledTrainingCommand, + createCommandProfileTrainer, type ImproveCandidateValidationInput, type ImproveCandidateValidator, type ImproveCodeBaseOptions, @@ -141,7 +144,13 @@ export { type ImproveScenarioPartitions, type ImproveSkillsOptions, type ImproveSurface, + type ImproveTrainingOptions, + type ImproveTrainingResult, improve, + type ProfileTrainer, + type ProfileTrainerRequest, + type TrainingBoundaryResult, + type TrainingDatasetDocument, } from './improvement/improve' export type { CandidateGenerator } from './improvement/improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './improvement/mcp-serve-verifier' @@ -163,6 +172,7 @@ export { createProfileImprovementHarness, type ProfileImprovementHarness, type ProfileImprovementHarnessRunOptions, + type ProfileImprovementHarnessTrainOptions, } from './improvement/profile-improvement-harness' export type { DeepReadonly, ReadonlyAgentProfile } from './improvement/profile-types' export { diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index f03e2455..5727a5fa 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:b5fc58845c940d8f7c61bc9cf120b01b76b9d085a5490a1a978844a4134c0616", + "digest": "sha256:224843e066e78674681eb5334f0cf291d30037c92727a5bf6d5a219109922f3e", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.241.1" + "runtimeVersion": "0.242.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:a7cce9d8e6b336c10989e74a75d160715b3a9eb1a2ff0ba0b2e1584f54dc62aa", - "runId": "agent-runtime-0.241.1-proposal-fixture", + "recordDigest": "sha256:91b24d43247fc2688b49735c25914a44a3e4ce2421337433837796cea8f01542", + "runId": "agent-runtime-0.242.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.241.1-proposal-fixture" + "runId": "agent-runtime-0.242.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index c1ce2fa0..8dd9fd41 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:23b992c8fc4146b76e2704f3ba38dc7b13408150439a112b2be9e73fc591aca2", + "digest": "sha256:97be4c92577f9497ce4d2897353c43cd07f29e94126b2e928e50cc66fba33113", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.241.1" + "runtimeVersion": "0.242.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:2455e1f27e26f1767fd3ad8c78fa00ab7a3928f8268d4b4b3d14dcef75a03270", + "recordDigest": "sha256:951ee09c461d8007f03a42a3d1f58510e877a5e097843b8c4138c241af121cdb", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/profile-training-boundaries.test.ts b/tests/profile-training-boundaries.test.ts new file mode 100644 index 00000000..bf25595c --- /dev/null +++ b/tests/profile-training-boundaries.test.ts @@ -0,0 +1,277 @@ +import assert from 'node:assert/strict' +import { chmodSync, readFileSync, writeFileSync } from 'node:fs' +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { type AgentProfile, sha256Bytes, sha256Utf8 } from '@tangle-network/agent-interface' +import { afterEach, describe, it, vi } from 'vitest' +import { + type CheckpointServingPort, + type ImproveTrainingOptions, + type ProfileTrainer, + runProfileTraining, + type TrainingDatasetDocument, +} from '../src/improvement/training' + +const faults = vi.hoisted(() => ({ + afterWrite: undefined as ((path: string) => void) | undefined, + afterSync: undefined as ((path: string) => void) | undefined, + afterPublication: undefined as ((path: string) => void) | undefined, + publications: [] as string[], + syncedDirectories: [] as string[], +})) + +// Keep real files and real fsyncs; inject cancellation at the exact I/O boundary. +vi.mock('node:fs/promises', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + async writeFile(...args: Parameters) { + await actual.writeFile(...args) + faults.afterWrite?.(String(args[0])) + }, + async open(...args: Parameters) { + const handle = await actual.open(...args) + const sync = handle.sync.bind(handle) + handle.sync = async () => { + await sync() + faults.afterSync?.(String(args[0])) + } + return handle + }, + } +}) + +vi.mock('../src/runtime/supervise/durable-file', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + syncDurableDirectory(path: string) { + actual.syncDurableDirectory(path) + faults.syncedDirectories.push(path) + }, + publishExclusiveDurableFile(...args: Parameters) { + const published = actual.publishExclusiveDurableFile(...args) + if (published) faults.publications.push(args[0]) + faults.afterPublication?.(args[0]) + return published + }, + } +}) + +let root: string | undefined + +afterEach(async () => { + faults.afterWrite = undefined + faults.afterSync = undefined + faults.afterPublication = undefined + faults.publications.length = 0 + faults.syncedDirectories.length = 0 + if (root) await rm(root, { recursive: true, force: true }) + root = undefined +}) + +async function fixture() { + root = await mkdtemp(join(tmpdir(), 'training-boundaries-')) + const controller = new AbortController() + const profile: AgentProfile = { + name: 'training-boundary-fixture', + harness: 'opencode', + model: { provider: 'openai-compat', default: 'base-model' }, + } + const document: TrainingDatasetDocument = { + version: 1, + format: 'sft', + rows: [ + { + task: { benchmark: 'fixture', task: 'train', contentDigest: sha256Utf8('task') }, + partition: 'train', + data: { messages: [{ role: 'assistant', content: 'captured transcript' }] }, + }, + ], + } + const bytes = Buffer.from(JSON.stringify(document)) + const path = join(root, 'source.json') + await writeFile(path, bytes) + const execute = vi.fn(async (request) => { + await writeFile(request.checkpointPath, 'weights') + return { succeeded: true, value: undefined } + }) + const serve = vi.fn(async (input) => ({ + succeeded: true, + value: { + artifactDigest: sha256Bytes(await readFile(input.artifactPath)), + routerModelId: input.routerModelId, + evidenceDigest: sha256Utf8('serving evidence'), + }, + })) + const options: ImproveTrainingOptions = { + mode: 'training', + trainer: { + identity: { mode: 'managed', id: 'fixture', revision: sha256Utf8('trainer') }, + execute, + }, + dataset: { path, digest: sha256Bytes(bytes) }, + parameters: {}, + executionRef: sha256Utf8('bound execution'), + serving: { serve }, + outputDirectory: join(root, 'outputs'), + maxCheckpointBytes: 1024, + timeoutMs: 10_000, + signal: controller.signal, + } + return { profile, options, controller, execute, serve } +} + +async function assertNoProfile(outputDirectory: string | undefined) { + if (outputDirectory) + await assert.rejects(readFile(join(outputDirectory, 'profile.json')), { code: 'ENOENT' }) +} + +describe('training cancellation, checkpoint integrity and publication', () => { + it('does not dispatch training when cancellation arrives during input persistence', async () => { + const { profile, options, controller, execute, serve } = await fixture() + faults.afterWrite = (path) => { + if (path.endsWith('parent-profile.json')) controller.abort(new Error('cancelled in I/O')) + } + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(execute.mock.calls.length, 0) + assert.equal(serve.mock.calls.length, 0) + assert.equal(result.trainingMayExist, false) + assert.equal(result.servingMayExist, false) + assert.match(result.reason, /cancelled in I\/O/) + await assertNoProfile(result.outputDirectory) + }) + + it('does not deploy after cancellation during checkpoint fsync', async () => { + const { profile, options, controller, execute, serve } = await fixture() + faults.afterSync = (path) => { + if (path.endsWith('checkpoint.bin')) controller.abort(new Error('cancelled before serving')) + } + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(execute.mock.calls.length, 1) + assert.equal(serve.mock.calls.length, 0) + assert.equal(result.stage, 'serving') + assert.equal(result.trainingMayExist, false) + assert.equal(result.servingMayExist, false) + await assertNoProfile(result.outputDirectory) + }) + + it('rejects checkpoint mutation by the candidate validator', async () => { + const { profile, options, serve } = await fixture() + options.validateCandidate = ({ isBaseline }) => { + if (isBaseline) return + const path = serve.mock.calls[0]![0].artifactPath + chmodSync(path, 0o600) + writeFileSync(path, 'different weights') + } + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(result.stage, 'profile') + assert.match(result.reason, /checkpoint changed/) + assert.equal(result.servingMayExist, true) + await assertNoProfile(result.outputDirectory) + }) + + it('rejects checkpoint mutation by the serving adapter', async () => { + const { profile, options, serve } = await fixture() + serve.mockImplementation(async (input) => { + chmodSync(input.artifactPath, 0o600) + writeFileSync(input.artifactPath, 'different weights') + return { + succeeded: true, + value: { + artifactDigest: input.artifactDigest, + routerModelId: input.routerModelId, + evidenceDigest: sha256Utf8('evidence'), + }, + } + }) + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(result.stage, 'serving') + assert.match(result.reason, /checkpoint changed/) + await assertNoProfile(result.outputDirectory) + }) + + it('retains uncertainty when a serving adapter ignores cancellation', async () => { + const { profile, options, controller, serve } = await fixture() + serve.mockImplementation(() => { + controller.abort(new Error('cancelled during serving')) + return new Promise(() => {}) + }) + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(result.stage, 'serving') + assert.equal(result.servingMayExist, true) + assert.equal(result.trainingMayExist, false) + assert.match(result.reason, /cancelled during serving/) + await assertNoProfile(result.outputDirectory) + }) + + it('does not invoke either port for an already cancelled request', async () => { + const { profile, options, controller, execute, serve } = await fixture() + controller.abort(new Error('already cancelled')) + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(result.stage, 'admission') + assert.equal(execute.mock.calls.length, 0) + assert.equal(serve.mock.calls.length, 0) + assert.equal(result.outputDirectory, undefined) + }) + + it('contains synchronous adapter failures without publishing a profile', async () => { + const { profile, options, execute } = await fixture() + execute.mockImplementation(() => { + throw new Error('adapter dispatch failed') + }) + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(result.stage, 'training') + assert.match(result.reason, /adapter dispatch failed/) + assert.equal(result.trainingMayExist, true) + await assertNoProfile(result.outputDirectory) + }) + + it('publishes the durable receipt before the profile through the shared writer', async () => { + const { profile, options } = await fixture() + faults.afterPublication = (path) => { + if (path.endsWith('profile.json')) { + const receipt = JSON.parse(readFileSync(join(dirname(path), 'receipt.json'), 'utf8')) + assert.equal(receipt.dataset.digest, options.dataset.digest) + } + } + const result = await runProfileTraining(profile, options) + assert(result.succeeded, JSON.stringify(result)) + assert.deepEqual(faults.publications, [result.receiptPath, result.profilePath]) + assert(faults.syncedDirectories.includes(options.outputDirectory)) + assert.equal((await stat(result.profilePath)).mode & 0o777, 0o400) + assert.equal((await stat(dirname(result.profilePath))).mode & 0o777, 0o700) + }) + + it('does not retract a committed profile for cancellation after publication', async () => { + const { profile, options, controller } = await fixture() + faults.afterPublication = (path) => { + if (path.endsWith('profile.json')) controller.abort(new Error('cancelled after commit')) + } + const result = await runProfileTraining(profile, options) + assert(controller.signal.aborted) + assert(result.succeeded, JSON.stringify(result)) + assert.deepEqual(JSON.parse(await readFile(result.profilePath, 'utf8')), result.profile) + }) + + it('durably withdraws the profile when publication reports a storage failure', async () => { + const { profile, options } = await fixture() + faults.afterPublication = (path) => { + if (path.endsWith('profile.json')) throw new Error('publication storage failure') + } + const result = await runProfileTraining(profile, options) + assert(!result.succeeded) + assert.equal(result.stage, 'persistence') + assert.match(result.reason, /publication storage failure/) + assert.equal(faults.syncedDirectories.at(-1), result.outputDirectory) + await assertNoProfile(result.outputDirectory) + }) +}) diff --git a/tests/profile-training.test.ts b/tests/profile-training.test.ts new file mode 100644 index 00000000..e25f8b42 --- /dev/null +++ b/tests/profile-training.test.ts @@ -0,0 +1,396 @@ +import assert from 'node:assert/strict' +import { mkdtemp, readdir, readFile, realpath, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setTimeout as sleep } from 'node:timers/promises' +import { + type AgentProfile, + canonicalAgentProfileDigest, + canonicalCandidateDigest, + sha256Bytes, + sha256Utf8, + trainedModelIdForArtifact, +} from '@tangle-network/agent-interface' +import { describe, it } from 'vitest' +import { improve } from '../src/improvement/improve' +import { createProfileImprovementHarness } from '../src/improvement/profile-improvement-harness' +import { + type CheckpointServingPort, + createCommandProfileTrainer, + type ImproveTrainingOptions, + type ProfileTrainer, + type TrainingDatasetDocument, +} from '../src/improvement/training' + +// A CPU-only learned scalar fixture. This proves training execution, not agent quality. +const TRAIN = ` +const fs = require('node:fs'); +let input = ''; process.stdin.on('data', b => input += b); +process.stdin.on('end', () => { + const r = JSON.parse(input); + if (process.env.P5_PRIVATE_CANARY) throw new Error('ambient environment leaked'); + const d = JSON.parse(fs.readFileSync(r.datasetPath, 'utf8')); + let weight = 0; + for (let epoch = 0; epoch < r.parameters.epochs; epoch++) { + for (const row of d.rows.filter(row => row.partition === 'train')) { + weight -= r.parameters.learningRate * 2 * row.data.x * (weight * row.data.x - row.data.y); + } + } + fs.writeFileSync(r.checkpointPath, JSON.stringify({ weight })); +}); +` + +const parent = (): AgentProfile => ({ + name: 'coder', + version: '1', + harness: 'opencode', + model: { provider: 'openai-compat', default: 'base-coder' }, + prompt: { instructions: ['Use the tools and build the artifact.'] }, +}) + +const serve: CheckpointServingPort = { + async serve(input) { + const digest = sha256Bytes(await readFile(input.artifactPath)) + assert.equal(digest, input.artifactDigest) + assert.equal(input.routerModelId, trainedModelIdForArtifact(digest)) + return { + succeeded: true, + value: { + artifactDigest: digest, + routerModelId: input.routerModelId, + evidenceDigest: canonicalCandidateDigest({ fixture: 'independent-serving-port', digest }), + }, + } + }, +} + +async function withFixture( + run: (options: ImproveTrainingOptions, dir: string) => Promise, + script = TRAIN, +): Promise { + const dir = await mkdtemp(join(tmpdir(), 'profile-training-test-')) + try { + const executable = await realpath(process.execPath) + const scriptPath = join(dir, 'trainer.cjs') + await writeFile(scriptPath, script) + const dataset: TrainingDatasetDocument = { + version: 1, + format: 'sft', + rows: [ + { + task: { benchmark: 'fixture', task: 'train', contentDigest: sha256Utf8('train') }, + partition: 'train', + data: { x: 1, y: 2 }, + }, + { + task: { + benchmark: 'fixture', + task: 'development', + contentDigest: sha256Utf8('development'), + }, + partition: 'validation', + data: { x: 2, y: 4 }, + }, + ], + } + const bytes = Buffer.from(JSON.stringify(dataset)) + const path = join(dir, 'source-dataset.json') + await writeFile(path, bytes) + const trainer = createCommandProfileTrainer({ + id: 'cpu-test-trainer', + executable: { path: executable, digest: sha256Bytes(await readFile(executable)) }, + args: [scriptPath], + inputs: [{ path: scriptPath, digest: sha256Utf8(script) }], + environment: {}, + maxOutputBytes: 4096, + }) + await run( + { + mode: 'training', + trainer, + dataset: { path, digest: sha256Bytes(bytes) }, + parameters: { epochs: 50, learningRate: 0.1 }, + executionRef: sha256Utf8('test-execution'), + serving: serve, + outputDirectory: join(dir, 'outputs'), + timeoutMs: 10_000, + maxCheckpointBytes: 4096, + }, + dir, + ) + } finally { + await rm(dir, { recursive: true, force: true }) + } +} + +async function assertNoProfile(outputDirectory: string): Promise { + const names = await readdir(outputDirectory).catch(() => []) + for (const name of names) + assert(!(await readdir(join(outputDirectory, name))).includes('profile.json')) +} + +describe('checkpoint training through improve', () => { + it('executes a pinned command and persists a receipt before a frozen profile', async () => { + await withFixture(async (options) => { + const original = parent() + const originalDigest = canonicalAgentProfileDigest(original) + const result = await improve(original, options) + assert(result.succeeded, JSON.stringify(result)) + const learned = JSON.parse(await readFile(result.artifactPath, 'utf8')) as { weight: number } + assert(learned.weight > 1.99 && learned.weight < 2.01) + assert.equal(result.receipt.parentProfileDigest, originalDigest) + assert.equal(result.receipt.dataset.digest, options.dataset.digest) + assert.deepEqual( + result.receipt.dataset.tasks.map((task) => task.task), + ['development', 'train'], + ) + assert.deepEqual(result.receipt.trainer, { + ...options.trainer.identity, + parameters: options.parameters, + }) + assert.equal( + result.receipt.checkpoint.artifactDigest, + sha256Bytes(await readFile(result.artifactPath)), + ) + assert.equal(result.profile.model?.default, result.receipt.checkpoint.routerModelId) + assert.equal( + result.profileDigest, + canonicalAgentProfileDigest(result.profile as AgentProfile), + ) + assert.deepEqual(JSON.parse(await readFile(result.receiptPath, 'utf8')), result.receipt) + assert.deepEqual(JSON.parse(await readFile(result.profilePath, 'utf8')), result.profile) + assert(Object.isFrozen(result.profile.metadata?.training?.receipt)) + assert.equal(canonicalAgentProfileDigest(original), originalDigest) + assert(!('decision' in result), 'training must not fabricate a ship verdict') + }) + }) + + it('does not inherit ambient credentials', async () => { + const before = process.env.P5_PRIVATE_CANARY + process.env.P5_PRIVATE_CANARY = 'private-test-value' + try { + await withFixture(async (options) => { + assert((await improve(parent(), options)).succeeded) + }) + } finally { + if (before === undefined) delete process.env.P5_PRIVATE_CANARY + else process.env.P5_PRIVATE_CANARY = before + } + }) + + it('refuses a wrong dataset digest before invoking the trainer', async () => { + await withFixture(async (options) => { + let invoked = false + options.trainer = { + identity: options.trainer.identity, + async execute() { + invoked = true + return { succeeded: true, value: undefined } + }, + } + options.dataset.digest = sha256Utf8('wrong') + const result = await improve(parent(), options) + assert(!result.succeeded) + assert.equal(result.stage, 'dataset') + assert.equal(invoked, false) + await assertNoProfile(options.outputDirectory) + }) + }) + + for (const [name, script] of [ + ['nonzero exit', `process.stdin.resume(); process.stdin.on('end', () => process.exit(7));`], + ['missing checkpoint', `process.stdin.resume();`], + ['empty checkpoint', TRAIN.replace('JSON.stringify({ weight })', "''")], + [ + 'symlink checkpoint', + TRAIN.replace( + 'fs.writeFileSync(r.checkpointPath, JSON.stringify({ weight }));', + 'fs.symlinkSync(r.datasetPath, r.checkpointPath);', + ), + ], + [ + 'excessive output', + `process.stdin.resume(); process.stdin.on('end', () => console.log('x'.repeat(100000)));`, + ], + [ + 'modified dataset', + TRAIN.replace( + 'let weight = 0;', + "fs.chmodSync(r.datasetPath, 0o600); fs.writeFileSync(r.datasetPath, '{}'); let weight = 0;", + ), + ], + ]) { + it(`refuses ${name} without producing a runnable profile`, async () => { + await withFixture(async (options) => { + const result = await improve(parent(), options) + assert(!result.succeeded, name) + await assertNoProfile(options.outputDirectory) + }, script) + }) + } + + it('refuses changed trainer inputs', async () => { + await withFixture(async (options, dir) => { + await writeFile(join(dir, 'trainer.cjs'), `${TRAIN}\n// changed after pinning`) + const result = await improve(parent(), options) + assert(!result.succeeded) + assert.match(result.reason, /digest mismatch/) + await assertNoProfile(options.outputDirectory) + }) + }) + + for (const kind of ['unverified', 'wrong-artifact', 'mutable-route'] as const) { + it(`refuses ${kind} serving evidence`, async () => { + await withFixture(async (options) => { + options.serving = { + async serve(input) { + if (kind === 'unverified') return { succeeded: false, reason: 'route not verified' } + return { + succeeded: true, + value: { + artifactDigest: + kind === 'wrong-artifact' ? sha256Utf8('other') : input.artifactDigest, + routerModelId: kind === 'mutable-route' ? 'fine-tune/latest' : input.routerModelId, + evidenceDigest: sha256Utf8('evidence'), + }, + } + }, + } + const result = await improve(parent(), options) + assert(!result.succeeded) + assert.equal(result.stage, 'serving') + await assertNoProfile(options.outputDirectory) + }) + }) + } + + it('bounds a managed trainer and reports unconfirmed remote cleanup', async () => { + await withFixture(async (options) => { + const managed: ProfileTrainer = { + identity: { mode: 'managed', id: 'managed-test', revision: sha256Utf8('adapter') }, + execute: async () => new Promise(() => {}), + } + const result = await improve(parent(), { ...options, trainer: managed, timeoutMs: 500 }) + assert(!result.succeeded) + assert.equal(result.stage, 'training') + assert.equal(result.trainingMayExist, true) + await assertNoProfile(options.outputDirectory) + }) + }) + + it('cancels the command process group including descendants', async () => { + await withFixture( + async (options, dir) => { + const checkpointPath = join(dir, 'cancel-checkpoint') + const controller = new AbortController() + const pending = options.trainer.execute( + { + version: 1, + invocationId: 'cancel-test', + datasetPath: options.dataset.path, + checkpointPath, + parentProfilePath: options.dataset.path, + parentProfileDigest: canonicalAgentProfileDigest(parent()), + parameters: {}, + executionRef: options.executionRef, + }, + controller.signal, + ) + try { + let ready = false + for (let i = 0; i < 100 && !ready; i++) { + ready = await readFile(`${checkpointPath}.ready`).then( + () => true, + () => false, + ) + if (!ready) await sleep(20) + } + assert(ready, 'the parent must have actually spawned before cancellation') + controller.abort() + assert.equal((await pending).succeeded, false) + await sleep(700) + await assert.rejects(() => readFile(`${checkpointPath}.late`), { code: 'ENOENT' }) + } finally { + controller.abort() + await pending + } + }, + ` +const fs = require('node:fs'), { spawn } = require('node:child_process'); +let input = ''; process.stdin.on('data', b => input += b); +process.stdin.on('end', () => { + const r = JSON.parse(input); + spawn(process.execPath, ['-e', "setTimeout(() => require('node:fs').writeFileSync(process.argv[1], 'late'), 600)", r.checkpointPath + '.late'], { stdio: 'ignore' }); + fs.writeFileSync(r.checkpointPath + '.ready', 'ready'); + setInterval(() => {}, 1000); +}); +`, + ) + }) + + it('uses the bound harness parent identity execution reference and validator', async () => { + await withFixture(async (options) => { + const original = parent() + const validations: boolean[] = [] + const harness = createProfileImprovementHarness({ + profile: original, + executionRef: sha256Utf8('bound-executor'), + agent: async () => { + throw new Error('training must not execute a benchmark task') + }, + validateCandidate: (input) => { + validations.push(input.isBaseline) + }, + }) + original.name = 'mutated-after-binding' + const result = await harness.train(options) + assert(result.succeeded, JSON.stringify(result)) + assert.equal(result.receipt.parentProfileDigest, harness.profileDigest) + assert.equal(result.receipt.executionRef, harness.executionRef) + assert.deepEqual(validations, [true, false]) + }) + }) + + it('retains the complete ancestry when training a trained parent', async () => { + await withFixture(async (options) => { + const first = await improve(parent(), options) + assert(first.succeeded) + const second = await improve(first.profile as AgentProfile, { + ...options, + parameters: { epochs: 30, learningRate: 0.1 }, + }) + assert(second.succeeded, JSON.stringify(second)) + assert.equal(second.receipt.parentProfileDigest, first.profileDigest) + assert.equal(second.receipt.parentReceiptDigest, canonicalCandidateDigest(first.receipt)) + assert.deepEqual(second.profile.metadata?.training?.ancestors, [first.receipt]) + }) + }) + + it('refuses training validation overlap before execution', async () => { + await withFixture(async (options) => { + const dataset = JSON.parse( + await readFile(options.dataset.path, 'utf8'), + ) as TrainingDatasetDocument + dataset.rows[1]!.task = dataset.rows[0]!.task + const bytes = Buffer.from(JSON.stringify(dataset)) + await writeFile(options.dataset.path, bytes) + options.dataset.digest = sha256Bytes(bytes) + const result = await improve(parent(), options) + assert(!result.succeeded) + assert.equal(result.stage, 'dataset') + assert.match(result.reason, /partitions intersect/) + }) + }) + + it('refuses a candidate rejected by the existing validation hook', async () => { + await withFixture(async (options) => { + options.validateCandidate = ({ isBaseline }) => { + if (!isBaseline) throw new Error('candidate refused') + } + const result = await improve(parent(), options) + assert(!result.succeeded) + assert.equal(result.stage, 'profile') + await assertNoProfile(options.outputDirectory) + }) + }) +})