Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions packages/fold-core/src/AgentRuntime/AgentRuntimeLayer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,20 @@ const assistantResultText = (message: Prompt.AssistantMessage): string | null =>
return text.length > 0 ? text : null
}

/**
* The tool as handed to `streamText`, which accepts any params while it reads the model's reply.
*
* With tool resolution disabled, `streamText` decodes each tool call's params against
* `tool.setParameters(Schema.toEncoded(tool.parametersSchema))`, so one bad call fails the whole reply and
* ends the run. Fold decodes params when it runs the tool (`toolkit.handle`), which turns a bad call into a
* failed tool result the model can fix. Only `setParameters` changes: providers build the tool's JSON
* schema from the unchanged `parametersSchema`, so the model sees the same schema.
*/
const acceptAnyParamsInReply = (tool: AnyTool): AnyTool =>
Object.assign(Object.create(Object.getPrototypeOf(tool)), tool, {
setParameters: () => tool.setParameters(Schema.Unknown),
})

/** Live AgentRuntime layer wiring the EventLog, hooks, model, and tool settlement into the loop. */
export const liveAgentRuntimeLayer: Layer.Layer<
AgentRuntime,
Expand Down Expand Up @@ -356,9 +370,9 @@ export const liveAgentRuntimeLayer: Layer.Layer<
// toolkit is filtered to the projected active names, so a tools-change entry binds the very
// next request. Handlers stay untouched - settlement still executes against the full Toolset.
const withHandler = yield* toolset.withHandler
const activeToolEntries = Object.entries(withHandler.tools).filter(([name]) =>
runtimeState.activeTools.includes(name),
)
const activeToolEntries = Object.entries(withHandler.tools)
.filter(([name]) => runtimeState.activeTools.includes(name))
.map(([name, tool]) => [name, acceptAnyParamsInReply(tool)] as const)
const toolkit: Toolkit.WithHandler<Record<string, AnyTool>> = {
tools: Object.fromEntries(activeToolEntries),
handle: withHandler.handle,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { expect, it } from '@effect/vitest'
import { Predicate, Effect, Ref } from 'effect'
import { Tool } from 'effect/ai'

import { AgentRuntime, type ToolResultLogEntry } from '../../src/index'
import { makeScriptedLanguageModel, textTurn, toolCallTurn } from '../TestLayers/ScriptedLanguageModel'
import { layerEchoTool, makeEchoRecorder } from '../TestLayers/TestTools'
import { collectEntries } from '../ToolRuntime/ToolRuntimeTestHelpers'
import { agentRuntimeBaseLayer, runInput, startInput } from './AgentRuntimeTestHelpers'

it.effect('returns a tool call with bad params to the model as a failed tool result', () =>
Effect.gen(function* () {
const recorder = yield* makeEchoRecorder
const scripted = yield* makeScriptedLanguageModel([
toolCallTurn([{ id: 'provider-call-1', name: 'echo', params: { txet: 'hi' } }]),
textTurn('Fixed my call'),
])
const layer = agentRuntimeBaseLayer(scripted.layer, layerEchoTool(recorder))

const result = yield* Effect.gen(function* () {
const runtime = yield* AgentRuntime

yield* runtime.start(startInput())
const finished = yield* runtime.run(runInput('use the echo tool'))
const entries = yield* collectEntries
const calls = yield* Ref.get(recorder.calls)

return { finished, entries, calls }
}).pipe(Effect.provide(layer))

expect(result.calls).toEqual([])
expect(result.finished.outcome).toBe('completed')
expect(result.finished.resultText).toBe('Fixed my call')

const toolResult = result.entries.find((entry): entry is ToolResultLogEntry =>
Predicate.isTagged(entry, 'tool-result'),
)
const part = toolResult?.message.content.find((content) => content.type === 'tool-result')
expect(part).toMatchObject({
isFailure: true,
result: { reason: { _tag: 'ToolParameterValidationError', description: 'Missing key\n at ["text"]' } },
})

// The model is still offered the tool's real params schema.
const requests = yield* scripted.requests
const echo = requests[0]?.tools.find((tool) => tool.name === 'echo')
if (echo === undefined) throw new Error('expected the echo tool on the first request')
expect(Tool.getJsonSchema(echo)).toMatchObject({ properties: { text: { type: 'string' } }, required: ['text'] })
}),
)
5 changes: 4 additions & 1 deletion packages/fold-core/test/TestLayers/ScriptedLanguageModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import { AnthropicLanguageModel } from '@humanlayer/effect-ai-anthropic'
import { OpenAiLanguageModel } from '@humanlayer/effect-ai-openai'
import { Predicate, Effect, Layer, Ref, Stream } from 'effect'
import { AiError, LanguageModel, type Prompt, type Response } from 'effect/ai'
import { AiError, LanguageModel, type Prompt, type Response, type Tool } from 'effect/ai'

/** Optional shaping for a scripted turn's finish part. */
export type ScriptedFinishOptions = {
Expand Down Expand Up @@ -89,6 +89,8 @@ export type ScriptedRequest = {
readonly prompt: Prompt.Prompt
/** Tool names advertised to the model on this request. */
readonly toolNames: ReadonlyArray<string>
/** The tools as the provider received them, to build the schemas it advertises. */
readonly tools: ReadonlyArray<Tool.Any>
/** The OpenAI per-request Config in context at request time, or null when none was provided. */
readonly openAiConfig: typeof OpenAiLanguageModel.Config.Service | null
/** The Anthropic per-request Config in context at request time, or null when none was provided. */
Expand Down Expand Up @@ -136,6 +138,7 @@ export const makeScriptedLanguageModel = (turns: ReadonlyArray<ScriptedTurn>): E
{
prompt: options.prompt,
toolNames: options.tools.map((tool) => tool.name),
tools: options.tools,
openAiConfig: Predicate.isTagged(openAiConfig, 'Some') ? openAiConfig.value : null,
anthropicConfig: Predicate.isTagged(anthropicConfig, 'Some') ? anthropicConfig.value : null,
},
Expand Down
Loading