Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 95 additions & 1 deletion src/runtime/environment-provider.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval'
import { estimateCost, HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval'
import {
type AgentEnvironmentCapabilities,
type AgentExactRunControlRef,
Expand Down Expand Up @@ -228,6 +228,80 @@ describe('environment provider adapters', () => {
},
)

it('prices unreceipted provider work from the catalog instead of settling a bare zero', async () => {
const executor = unreceiptedProviderExecutor('glm-5.3')
const events = await collect(
executor.execute('task', new AbortController().signal) as AsyncIterable<UsageEvent>,
)
const expected = estimateCost(200_000, 20_000, 'glm-5.3')
expect(expected).toBeGreaterThan(0)
expect(events).toContainEqual({
kind: 'cost',
usd: expected,
usdKnown: false,
usdEstimated: expected,
provenance: 'catalog-estimate',
})
const spent = executor.resultArtifact().spent
// A price is not a receipt, so the whole amount stays subtractable and no dollar is proven.
expect(spent.usdKnown).toBe(false)
expect(spent.usd).toBeCloseTo(expected)
expect(spent.usd - (spent.usdEstimated ?? 0)).toBe(0)
})

it('settles a model the catalog cannot price with no estimate at all', async () => {
const executor = unreceiptedProviderExecutor('model-with-no-catalog-entry')
const events = await collect(
executor.execute('task', new AbortController().signal) as AsyncIterable<UsageEvent>,
)
expect(events).toContainEqual({
kind: 'cost',
usd: 0,
usdKnown: false,
provenance: 'uncaptured',
})
const spent = executor.resultArtifact().spent
expect(spent.usd).toBe(0)
expect(spent.usdKnown).toBe(false)
// Absence, not a zero estimate: nothing was priced, rather than priced at nothing.
expect(spent.usdEstimated).toBeUndefined()
})

it('does not price an execution that already put billed dollars on the channel', async () => {
const provider: AgentEnvironmentProvider = {
name: 'partly-billed-fixture',
capabilities: fakeCapabilities,
create: async () =>
fakeEnvironment({
async *stream() {
yield {
type: 'llm_call',
data: {
tokensIn: 100_000,
tokensOut: 10_000,
costUsd: 0.03,
costProvenance: 'billing-receipt',
},
}
yield { type: 'llm_call', data: { tokensIn: 100_000, tokensOut: 10_000 } }
yield { type: 'done', data: { finalText: 'result' } }
},
}),
}
const signal = new AbortController().signal
const executor = providerAsExecutor(provider)(
{ profile: { name: 'worker', model: { default: 'glm-5.3' } }, harness: null },
{ signal, seams: {} },
)
await collect(executor.execute('task', signal) as AsyncIterable<UsageEvent>)
const spent = executor.resultArtifact().spent
// The token total is the whole execution's, not the unbilled remainder, so pricing it here
// would charge the first call's 100k prompt tokens a second time.
expect(spent.usd).toBeCloseTo(0.03)
expect(spent.usdEstimated).toBeUndefined()
expect(spent.usdKnown).toBe(false)
})

it('joins two independently refined provider snapshots without replacing the other worker', async () => {
const done: AgentEnvironmentEvent = {
type: 'done',
Expand Down Expand Up @@ -3749,6 +3823,26 @@ describe('declared provider placements', () => {
})
})

/** A provider that reports tokens and never a dollar — the sandbox-rooted shape that used to
* settle `usd: 0` with no estimate no matter how much prompt it had processed. */
function unreceiptedProviderExecutor(model: string) {
const provider: AgentEnvironmentProvider = {
name: 'unreceipted-fixture',
capabilities: fakeCapabilities,
create: async () =>
fakeEnvironment({
async *stream() {
yield { type: 'llm_call', data: { tokensIn: 200_000, tokensOut: 20_000 } }
yield { type: 'done', data: { finalText: 'result' } }
},
}),
}
return providerAsExecutor(provider)(
{ profile: { name: 'worker', model: { default: model } }, harness: null },
{ signal: new AbortController().signal, seams: {} },
)
}

function fakeEnvironment(
overrides: Partial<AgentEnvironment> & Pick<AgentEnvironment, 'stream'>,
): AgentEnvironment {
Expand Down
47 changes: 43 additions & 4 deletions src/runtime/environment-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ import {
} from './sandbox-events'
import type { SandboxOutcomeCarrier } from './sandbox-outcome'
import { linkAbort } from './supervise/abortable'
import { priceUnreceiptedWork } from './supervise/cost-estimate'
import {
attestRuntimeOwnedPendingExecutor,
finalizeRuntimeOwnedPendingExecutor,
Expand Down Expand Up @@ -998,6 +999,9 @@ async function* streamProviderExecutor(
let sawCostEstimate = false
let pendingUnpricedWork = false
let pendingEstimate: number | undefined
/** The model id a usage receipt reported, when it reported one. Only a catalog can price a run
* the provider never billed, and only a model id can address a catalog. */
let observedModel: string | undefined
let usdEstimated = 0
let usd = 0
let text = ''
Expand Down Expand Up @@ -1056,19 +1060,29 @@ async function* streamProviderExecutor(
// Interim missing prices and catalog estimates may be covered by a later cumulative bill.
// Only unresolved work becomes unknown at settlement; do not add a quote to that bill.
if (pendingUnpricedWork) {
const estimate = pendingEstimate ?? 0
// A provider that reports tokens and no dollars used to settle a bare `$0`, so a sandbox-
// rooted run that certainly spent money reported a dollar total of zero with no estimate at
// all. The catalog answers what the provider WOULD bill, so the amount rides `usdEstimated`
// and `usdKnown` stays false below: a price is not a receipt and must never become one.
//
// Priced only when NO dollars reached the channel for this execution. `tokens` is the
// execution's cumulative total, not the unresolved remainder, so pricing it alongside any
// receipt — billed or already marked unknown — would charge the same tokens twice. A
// partially billed execution therefore keeps its unresolved part unknown, which is true.
const estimated = pendingEstimate ?? (usd === 0 ? catalogPrice() : undefined)
const estimate = estimated ?? 0
sawUnknownCostReceipt = true
usd += estimate
if (pendingEstimate !== undefined) {
if (estimated !== undefined) {
sawCostEstimate = true
usdEstimated += estimate
}
yield {
kind: 'cost',
usd: estimate,
usdKnown: false,
...(pendingEstimate === undefined ? {} : { usdEstimated: estimate }),
provenance: pendingEstimate === undefined ? 'uncaptured' : 'catalog-estimate',
...(estimated === undefined ? {} : { usdEstimated: estimate }),
provenance: estimated === undefined ? 'uncaptured' : 'catalog-estimate',
}
}
if ((args.options.requireTerminalEvent ?? true) && !terminal) {
Expand Down Expand Up @@ -1208,11 +1222,36 @@ async function* streamProviderExecutor(
}
if (failed) throw failure

/**
* The catalog price of this execution's whole token total, under the first model id the catalog
* knows. Undefined when none prices, so an unpriced model settles with no estimate at all rather
* than a zero one — absence says "nothing could be priced", a zero would say "priced at nothing".
*
* Candidates run most specific first: the id a usage receipt reported, then the turn's own
* model, then the profile's. A provider that selected its own default declares no profile model,
* so a reported id is then the only one a catalog can match.
*/
function catalogPrice(): number | undefined {
for (const model of [observedModel, turn.model, concreteProfileModel(args.createProfile)]) {
if (model === undefined) continue
const priced = priceUnreceiptedWork({
inputTokens: tokens.input,
outputTokens: tokens.output,
model,
})
if (priced.usdKnown === false && priced.usdEstimated !== undefined) return priced.usdEstimated
}
return undefined
}

function* creditUsage(
receipt: ReturnType<typeof usageLedger.observe>,
event?: SandboxEvent,
): Iterable<UsageEvent> {
if (receipt === undefined) return
// A receipt that named no model is stamped with the worker's name, which is not a model id and
// prices nothing. Only a different value is evidence of what the provider actually served.
if (receipt.model !== (args.profile.name ?? 'agent')) observedModel = receipt.model
const hasTokens = receipt.tokensIn !== undefined || receipt.tokensOut !== undefined
if (
hasTokens &&
Expand Down
Loading