From 88b905f6cb96e9a1872341c3d5d2bd47d2a10517 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 18 Sep 2026 07:54:07 -0600 Subject: [PATCH] fix(provider-cost): price a sandbox execution the provider never billed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `priceUnreceiptedWork` had exactly one caller, the bridge executor. The provider/sandbox executor obtained a dollar figure only by copying `receipt.estimatedCostUsd` off the harness stream, so a provider that reported tokens and no dollars settled `usd: 0` with no `usdEstimated` at all — not even a catalog floor. Reproduced here: a provider fixture presenting 200,000 prompt and 20,000 completion tokens under a profile declaring `glm-5.3` settles `{ usd: 0, usdKnown: false, provenance: 'uncaptured' }`, while `estimateCost(200_000, 20_000, 'glm-5.3')` is 0.4416 and `isModelPriced('glm-5.3')` is true. The corpus effect — five sandbox-rooted discovery runs recording `usd=0` on 252k to 49.3M input tokens beside a bridge-rooted run recording $54.50 on 48.4M — is reported from that program's records and was not re-measured here. Settlement now asks the catalog when the harness supplied no estimate. Two limits hold the number honest: - `usdKnown` stays false and the whole amount rides `usdEstimated`, so `usd - usdEstimated` remains the dollars a provider is known to have billed. A price approximates what a provider would bill; it never measures what it did. - It prices only an execution with no dollars on the channel. `tokens` is the execution's cumulative total rather than the unresolved remainder, so pricing it beside any receipt would charge the same tokens twice — the same reason bridge-executor gates on `turnKnownCostSubtotal === 0`. A partly billed execution keeps its unresolved part unknown, which is true. A model the catalog cannot price still settles `usd: 0, usdKnown: false` with `usdEstimated` absent: nothing could be priced, rather than priced at nothing. Every sentinel that could reach the catalog by accident — the worker's name, `default`, `unknown` — is unpriced, so none can invent a dollar. Admission does not move. An unknown-dollar settlement already zeroed a usd-capped root's free balance and still does; what changes is the recorded total. Tests: a run with 200k/20k tokens on a priced model settles the catalog amount with `usdKnown: false` and `usd - usdEstimated === 0`; the same run on an unpriced model settles zero with no estimate; a run holding a billing receipt plus a later unbilled call keeps its $0.03 and gains no estimate. Partly serves #1175, which asks for three things: carry the provider's reported per-call cost with `usdKnown: true` (untouched here), keep a subscription route unpriced rather than catalog-estimated (in tension with this change, which prices any execution the provider did not bill), and keep an estimate separable from reported cost in the projection (which this satisfies). #1175 stays open. Co-Authored-By: Claude Opus 5 (1M context) --- src/runtime/environment-provider.test.ts | 96 +++++++++++++++++++++++- src/runtime/environment-provider.ts | 47 +++++++++++- 2 files changed, 138 insertions(+), 5 deletions(-) diff --git a/src/runtime/environment-provider.test.ts b/src/runtime/environment-provider.test.ts index 06cb0d1d..1079bcd1 100644 --- a/src/runtime/environment-provider.test.ts +++ b/src/runtime/environment-provider.test.ts @@ -1,4 +1,4 @@ -import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' +import { estimateCost, HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import { type AgentEnvironmentCapabilities, type AgentExactRunControlRef, @@ -228,6 +228,80 @@ describe('environment provider adapters', () => { }, ) + it('prices unreceipted provider work from the catalog instead of settling a bare zero', async () => { + const executor = unreceiptedProviderExecutor('glm-5.3') + const events = await collect( + executor.execute('task', new AbortController().signal) as AsyncIterable, + ) + const expected = estimateCost(200_000, 20_000, 'glm-5.3') + expect(expected).toBeGreaterThan(0) + expect(events).toContainEqual({ + kind: 'cost', + usd: expected, + usdKnown: false, + usdEstimated: expected, + provenance: 'catalog-estimate', + }) + const spent = executor.resultArtifact().spent + // A price is not a receipt, so the whole amount stays subtractable and no dollar is proven. + expect(spent.usdKnown).toBe(false) + expect(spent.usd).toBeCloseTo(expected) + expect(spent.usd - (spent.usdEstimated ?? 0)).toBe(0) + }) + + it('settles a model the catalog cannot price with no estimate at all', async () => { + const executor = unreceiptedProviderExecutor('model-with-no-catalog-entry') + const events = await collect( + executor.execute('task', new AbortController().signal) as AsyncIterable, + ) + expect(events).toContainEqual({ + kind: 'cost', + usd: 0, + usdKnown: false, + provenance: 'uncaptured', + }) + const spent = executor.resultArtifact().spent + expect(spent.usd).toBe(0) + expect(spent.usdKnown).toBe(false) + // Absence, not a zero estimate: nothing was priced, rather than priced at nothing. + expect(spent.usdEstimated).toBeUndefined() + }) + + it('does not price an execution that already put billed dollars on the channel', async () => { + const provider: AgentEnvironmentProvider = { + name: 'partly-billed-fixture', + capabilities: fakeCapabilities, + create: async () => + fakeEnvironment({ + async *stream() { + yield { + type: 'llm_call', + data: { + tokensIn: 100_000, + tokensOut: 10_000, + costUsd: 0.03, + costProvenance: 'billing-receipt', + }, + } + yield { type: 'llm_call', data: { tokensIn: 100_000, tokensOut: 10_000 } } + yield { type: 'done', data: { finalText: 'result' } } + }, + }), + } + const signal = new AbortController().signal + const executor = providerAsExecutor(provider)( + { profile: { name: 'worker', model: { default: 'glm-5.3' } }, harness: null }, + { signal, seams: {} }, + ) + await collect(executor.execute('task', signal) as AsyncIterable) + const spent = executor.resultArtifact().spent + // The token total is the whole execution's, not the unbilled remainder, so pricing it here + // would charge the first call's 100k prompt tokens a second time. + expect(spent.usd).toBeCloseTo(0.03) + expect(spent.usdEstimated).toBeUndefined() + expect(spent.usdKnown).toBe(false) + }) + it('joins two independently refined provider snapshots without replacing the other worker', async () => { const done: AgentEnvironmentEvent = { type: 'done', @@ -3749,6 +3823,26 @@ describe('declared provider placements', () => { }) }) +/** A provider that reports tokens and never a dollar — the sandbox-rooted shape that used to + * settle `usd: 0` with no estimate no matter how much prompt it had processed. */ +function unreceiptedProviderExecutor(model: string) { + const provider: AgentEnvironmentProvider = { + name: 'unreceipted-fixture', + capabilities: fakeCapabilities, + create: async () => + fakeEnvironment({ + async *stream() { + yield { type: 'llm_call', data: { tokensIn: 200_000, tokensOut: 20_000 } } + yield { type: 'done', data: { finalText: 'result' } } + }, + }), + } + return providerAsExecutor(provider)( + { profile: { name: 'worker', model: { default: model } }, harness: null }, + { signal: new AbortController().signal, seams: {} }, + ) +} + function fakeEnvironment( overrides: Partial & Pick, ): AgentEnvironment { diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index d6abcedf..f9eea64a 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -96,6 +96,7 @@ import { } from './sandbox-events' import type { SandboxOutcomeCarrier } from './sandbox-outcome' import { linkAbort } from './supervise/abortable' +import { priceUnreceiptedWork } from './supervise/cost-estimate' import { attestRuntimeOwnedPendingExecutor, finalizeRuntimeOwnedPendingExecutor, @@ -998,6 +999,9 @@ async function* streamProviderExecutor( let sawCostEstimate = false let pendingUnpricedWork = false let pendingEstimate: number | undefined + /** The model id a usage receipt reported, when it reported one. Only a catalog can price a run + * the provider never billed, and only a model id can address a catalog. */ + let observedModel: string | undefined let usdEstimated = 0 let usd = 0 let text = '' @@ -1056,10 +1060,20 @@ async function* streamProviderExecutor( // Interim missing prices and catalog estimates may be covered by a later cumulative bill. // Only unresolved work becomes unknown at settlement; do not add a quote to that bill. if (pendingUnpricedWork) { - const estimate = pendingEstimate ?? 0 + // A provider that reports tokens and no dollars used to settle a bare `$0`, so a sandbox- + // rooted run that certainly spent money reported a dollar total of zero with no estimate at + // all. The catalog answers what the provider WOULD bill, so the amount rides `usdEstimated` + // and `usdKnown` stays false below: a price is not a receipt and must never become one. + // + // Priced only when NO dollars reached the channel for this execution. `tokens` is the + // execution's cumulative total, not the unresolved remainder, so pricing it alongside any + // receipt — billed or already marked unknown — would charge the same tokens twice. A + // partially billed execution therefore keeps its unresolved part unknown, which is true. + const estimated = pendingEstimate ?? (usd === 0 ? catalogPrice() : undefined) + const estimate = estimated ?? 0 sawUnknownCostReceipt = true usd += estimate - if (pendingEstimate !== undefined) { + if (estimated !== undefined) { sawCostEstimate = true usdEstimated += estimate } @@ -1067,8 +1081,8 @@ async function* streamProviderExecutor( kind: 'cost', usd: estimate, usdKnown: false, - ...(pendingEstimate === undefined ? {} : { usdEstimated: estimate }), - provenance: pendingEstimate === undefined ? 'uncaptured' : 'catalog-estimate', + ...(estimated === undefined ? {} : { usdEstimated: estimate }), + provenance: estimated === undefined ? 'uncaptured' : 'catalog-estimate', } } if ((args.options.requireTerminalEvent ?? true) && !terminal) { @@ -1208,11 +1222,36 @@ async function* streamProviderExecutor( } if (failed) throw failure + /** + * The catalog price of this execution's whole token total, under the first model id the catalog + * knows. Undefined when none prices, so an unpriced model settles with no estimate at all rather + * than a zero one — absence says "nothing could be priced", a zero would say "priced at nothing". + * + * Candidates run most specific first: the id a usage receipt reported, then the turn's own + * model, then the profile's. A provider that selected its own default declares no profile model, + * so a reported id is then the only one a catalog can match. + */ + function catalogPrice(): number | undefined { + for (const model of [observedModel, turn.model, concreteProfileModel(args.createProfile)]) { + if (model === undefined) continue + const priced = priceUnreceiptedWork({ + inputTokens: tokens.input, + outputTokens: tokens.output, + model, + }) + if (priced.usdKnown === false && priced.usdEstimated !== undefined) return priced.usdEstimated + } + return undefined + } + function* creditUsage( receipt: ReturnType, event?: SandboxEvent, ): Iterable { if (receipt === undefined) return + // A receipt that named no model is stamped with the worker's name, which is not a model id and + // prices nothing. Only a different value is evidence of what the provider actually served. + if (receipt.model !== (args.profile.name ?? 'agent')) observedModel = receipt.model const hasTokens = receipt.tokensIn !== undefined || receipt.tokensOut !== undefined if ( hasTokens &&