diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index 165cd584113..cf026bf4f5b 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -9936,3 +9936,16 @@ export function PitchBookIcon(props: SVGProps) { ) } + +/** TypeSafe’s official mark from https://typesafe.ai. */ +export function TypeSafeIcon(props: SVGProps) { + return ( + + + + ) +} diff --git a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx index 23c80273d81..1c9e4b6b197 100644 --- a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx +++ b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx @@ -47,6 +47,7 @@ import { Callout } from 'fumadocs-ui/components/callout' | `MISTRAL_API_KEY` | Mistral | | `XAI_API_KEY_1` | xAI | | `KIMI_API_KEY_1` | Moonshot Kimi | +| `TYPESAFE_API_KEY_1` / `_2` / `_3` | TypeSafe Jev hosted key rotation | | `ZAI_API_KEY_1` | Z.ai | | `TOGETHER_API_KEY` | Together AI | | `FIREWORKS_API_KEY` | Fireworks AI | diff --git a/apps/docs/content/docs/workflows/blocks/agent.mdx b/apps/docs/content/docs/workflows/blocks/agent.mdx index 241e3ac4c80..f754555a240 100644 --- a/apps/docs/content/docs/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/workflows/blocks/agent.mdx @@ -31,6 +31,30 @@ For a custom cloud deployment, enter its provider prefix and model ID: `azure/my Ollama Cloud, OpenRouter, Fireworks, Together AI, Baseten, Ollama, vLLM, and LiteLLM load their available models from the configured provider. New models appear through that discovery without a Sim catalog release. You can also enter a namespaced ID directly, such as `ollama-cloud/deepseek-v4.1-flash`, `openrouter/provider/model`, or `ollama/my-local-model`. Provider prefixes are case-insensitive; the model ID after the prefix keeps its original casing. +### Jev evaluation models + +Select `jev-latest` from TypeSafe in the Agent model selector. Hosted Sim supplies a key and bills model usage through the normal credit system; workspace or organization BYOK keys override the hosted key without model charges. Self-hosted users enter their TypeSafe key in the block. Use `jev-1.13.0` to pin a version or `jev-preview` to follow preview releases. These models use **State** and **Questions** in place of conversational messages. State accepts text or a reference to a JSON object or array. Questions is a JSON object keyed by the answer names you want: + +```json +{ + "route": { + "type": "choice", + "instructions": "Which team should handle this request?", + "criteria": { "billing": "Payments and invoices", "support": "Product issues" } + }, + "urgency": { + "type": "score", + "instructions": "How urgent is the request?", + "criteria": ["Routine", "Soon", "Immediate"] + }, + "resolved": { "type": "noul", "instructions": "Has the request been resolved?" } +} +``` + +Read results from `` or expand an answer in the reference picker, such as ``. Each Choice answer includes `choice`, `probabilities`, and `confidence`; each Score answer includes `score`, `legend`, `probabilities`, and `confidence`; each Noul answer includes `noul`, a probability from 0 to 1. `content` contains the same answers as JSON text, and the standard model, token, timing, and cost outputs remain available. Use a Condition block to route on these results. + +Jev evaluates the supplied state in one request. Chat messages, files, tools, skills, conversation memory, response-format schemas, and chat model fallbacks are hidden for these models. Saved settings return when you switch back to a chat model. TypeSafe documents a 64,000-token total request limit and a 32,000-token limit for state plus the longest question. See [TypeSafe's model documentation](https://docs.typesafe.ai/models) and [question formats](https://docs.typesafe.ai/api). + ### Files Files for the model to read: images for a vision-capable model, or documents for text. Upload them on the block, or pass a file from an earlier block, such as an upload trigger or an [API](/workflows/blocks/api) response, with a connection tag. @@ -104,7 +128,7 @@ Some settings live under advanced, or appear only for models that support them: - **Max output tokens.** Caps the response length. Defaults to the model's full limit. - **Reasoning effort / Thinking level.** For models with extended reasoning, how much the model thinks before answering. Higher is more thorough but slower and costs more tokens. - **Prompt caching.** For Anthropic Claude models, reuses the system prompt and tool definitions between runs instead of re-reading them every time. Cached input costs a tenth of the normal rate, but writing the cache costs 1.25x, so leave it off for one-off runs and turn it on when the same agent runs repeatedly. The cache covers a prefix only if it reaches 1,024 tokens (2,048 on Haiku) — below that Anthropic ignores it and nothing changes. Entries expire after five minutes of no use. -- **API key.** Your key for the chosen provider. Hidden on hosted Sim, which supplies one. +- **API key.** Your key for the chosen provider. Hidden when hosted Sim supplies a key for the selected model, including Jev. - **Fallback models.** An ordered list of up to five models to try when the request to the selected model fails, whether the provider is overloaded, rate-limited, or down. Sim tries the 2nd choice, then the 3rd, and so on, once each, and `` reports the model that answered. On hosted Sim, hosted models use your workspace's BYOK or platform credentials; local and self-hosted installations may still require a key. A model that needs its own key takes it from a workspace environment variable you pick on the row; a model on the same provider as the selected model reuses the block's key. A stored row key stops applying when its key field is hidden. Providers that require family-specific credentials, such as Vertex, can only be fallbacks for a selected model of the same family. The Auto model cannot be a fallback. A fallback runs with the selected model's settings where its provider accepts them: temperature and max output tokens are clamped to the fallback's limits, and when the fallback has a reasoning effort, thinking level, or verbosity setting that the selected model's value does not fit, the row shows that field so you can pick a value for it; leave it empty and the provider's default applies. - **Retry on fail.** Retries the selected model after a failure, up to a maximum number of tries with a wait between them. When its tries run out, the fallback models are tried in order, once each, with no wait before the first of them. A fallback is never retried. See [Retries and fallbacks](#retries-and-fallbacks) for how recorded tool results are reused and when a tool can execute again. diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/byok/byok.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/byok/byok.tsx index b7b3e19d43f..28bf3a7a232 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/byok/byok.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/byok/byok.tsx @@ -39,6 +39,7 @@ import { SerperIcon, TinyFishIcon, TogetherIcon, + TypeSafeIcon, WizaIcon, xAIIcon, ZaiIcon, @@ -130,6 +131,13 @@ const PROVIDERS: (BYOKManagerProvider & { id: BYOKProviderId })[] = [ description: 'LLM calls', placeholder: 'sk-...', }, + { + id: 'typesafe', + name: 'TypeSafe', + icon: TypeSafeIcon, + description: 'Jev evaluation models', + placeholder: 'Enter your TypeSafe API key', + }, { id: 'fireworks', name: 'Fireworks', @@ -352,6 +360,7 @@ const PROVIDER_SECTIONS: BYOKProviderSection[] = [ 'cohere', 'xai', 'kimi', + 'typesafe', 'fireworks', 'together', 'baseten', diff --git a/apps/sim/blocks/agent-evaluation.test.ts b/apps/sim/blocks/agent-evaluation.test.ts new file mode 100644 index 00000000000..bd7160a56aa --- /dev/null +++ b/apps/sim/blocks/agent-evaluation.test.ts @@ -0,0 +1,225 @@ +/** @vitest-environment node */ +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + getEffectiveBlockOutputPaths, + getEffectiveBlockOutputs, + getEffectiveBlockOutputType, +} from '@/lib/workflows/blocks/block-outputs' +import { getBlockReferenceTags } from '@/lib/workflows/blocks/block-reference-tags' +import { evaluateSubBlockCondition } from '@/lib/workflows/subblocks/visibility' +import { AgentBlock } from '@/blocks/blocks/agent' +import { getAgentModelOptions, getModelOptions } from '@/blocks/utils' +import { getBaseModelProviders } from '@/providers/models' +import { Serializer } from '@/serializer' +import { useProvidersStore } from '@/stores/providers/store' +import type { BlockState } from '@/stores/workflows/workflow/types' + +const { mockGetBlock } = vi.hoisted(() => ({ mockGetBlock: vi.fn() })) + +vi.mock('@/blocks', () => ({ getBlock: mockGetBlock })) + +describe('Agent evaluation configuration', () => { + afterEach(resetEnvFlagsMock) + beforeEach(() => { + mockGetBlock.mockReturnValue(AgentBlock) + }) + + it.each(['jev-1.13.0', 'jev-latest', 'jev-preview'])( + 'shows native fields and credentials for %s', + (model) => { + const visible = AgentBlock.subBlocks + .filter((field) => evaluateSubBlockCondition(field.condition, { model })) + .map((field) => field.id) + expect(visible).toEqual(['model', 'apiKey', 'evaluationState', 'evaluationQuestions']) + } + ) + + it('keeps evaluation inputs configurable for a model reference', () => { + for (const field of AgentBlock.subBlocks.filter((field) => field.id.startsWith('evaluation'))) { + expect(evaluateSubBlockCondition(field.condition, { model: '' })).toBe(true) + } + }) + + it.each([false, true])('shows TypeSafe credentials only when needed, hosted=%s', (hosted) => { + setEnvFlags({ isHosted: hosted }) + const apiKey = AgentBlock.subBlocks.find((field) => field.id === 'apiKey')! + expect(evaluateSubBlockCondition(apiKey.condition, { model: 'jev-latest' })).toBe(!hosted) + }) + + it.each(['jev-1.13.0', '', '{{MODEL_ID}}'])( + 'exposes answers for %s in downstream selectors', + (model) => { + const values = { model: { value: model } } + expect(getEffectiveBlockOutputs('agent', values)).toHaveProperty('answers') + expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers') + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') + } + ) + + it('does not expose evaluation answers for a known chat model', () => { + expect(getEffectiveBlockOutputs('agent', { model: { value: 'gpt-4o' } })).not.toHaveProperty( + 'answers' + ) + }) + + it.each(['jev-1.13.0', ''])( + 'keeps answers accessible with a saved chat schema for %s', + (model) => { + const outputs = getEffectiveBlockOutputs('agent', { + model: { value: model }, + responseFormat: { + value: { schema: { type: 'object', properties: { title: { type: 'string' } } } }, + }, + }) + expect(outputs).toHaveProperty('answers') + if (model === 'jev-1.13.0') expect(outputs).not.toHaveProperty('title') + else expect(outputs).toHaveProperty('title') + } + ) + + it('shows Jev only in the model picker that supports evaluation inputs', () => { + useProvidersStore.getState().setProviderModels('base', Object.keys(getBaseModelProviders())) + expect(getAgentModelOptions().map((option) => option.id)).toContain('jev-1.13.0') + expect(getModelOptions().map((option) => option.id)).not.toContain('jev-1.13.0') + }) + + describe('evaluation answer references', () => { + const questions = { + category: { type: 'choice', instructions: 'Choose a category', criteria: { a: 'A', b: 'B' } }, + rating: { type: 'score', instructions: 'Rate the result', criteria: ['Low', 'High'] }, + passed: { type: 'noul', instructions: 'Did it pass?' }, + } + + it.each(['jev-1.13.0', 'jev-latest', 'jev-preview', '', '{{MODEL_ID}}'])( + 'exposes typed question fields for %s without an execution result', + (model) => { + const values = { + model: { value: model }, + evaluationQuestions: { value: JSON.stringify(questions) }, + responseFormat: { + value: { schema: { type: 'object', properties: { title: { type: 'string' } } } }, + }, + } + const tags = getBlockReferenceTags({ + block: { id: 'agent-test', type: 'agent', name: 'Evaluate', subBlocks: values }, + }) + const fields = { + 'answers.category.choice': 'string', + 'answers.category.confidence': 'number', + 'answers.category.probabilities': 'json', + 'answers.category.type': 'string', + 'answers.rating.score': 'number', + 'answers.rating.confidence': 'number', + 'answers.rating.legend': 'json', + 'answers.passed.noul': 'number', + } + for (const [path, type] of Object.entries(fields)) { + expect(tags).toContain(`evaluate.${path}`) + expect(getEffectiveBlockOutputType('agent', path, values)).toBe(type) + } + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') + expect(getEffectiveBlockOutputType('agent', 'answers.category', values)).toBe('json') + expect(tags).not.toContain('evaluate.answers.passed.confidence') + expect(tags.includes('evaluate.title')).toBe(!model.startsWith('jev-')) + } + ) + + it.each([undefined, '', '{', '', '{{QUESTIONS}}', [], null, { unknown: {} }])( + 'keeps the answers object selectable when questions cannot be inferred: %j', + (value) => { + const values = { model: { value: 'jev-latest' }, evaluationQuestions: { value } } + expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers') + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') + } + ) + + it('uses structured questions and follows edits without leaking fields into chat models', () => { + const values = { + model: { value: 'jev-latest' }, + evaluationQuestions: { value: { result: questions.category } }, + } + expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers.result.choice') + expect( + getEffectiveBlockOutputPaths('agent', { + ...values, + evaluationQuestions: { value: { result: questions.passed } }, + }) + ).not.toContain('answers.result.choice') + expect( + getEffectiveBlockOutputPaths('agent', { + ...values, + model: { value: 'gpt-4o' }, + }).some((path) => path.startsWith('answers')) + ).toBe(false) + }) + + it('does not offer ambiguous reference paths for special question IDs', () => { + const values = { + model: { value: 'jev-latest' }, + evaluationQuestions: { + value: { + 'with.dot': questions.passed, + 'with space': questions.passed, + 'with[0]': questions.passed, + '': questions.passed, + 'valid-id_1': questions.passed, + }, + }, + } + const paths = getEffectiveBlockOutputPaths('agent', values) + expect(paths.filter((path) => path.startsWith('answers.'))).toEqual([ + 'answers.valid-id_1.noul', + 'answers.valid-id_1.type', + ]) + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') + }) + + it.each(['type', 'properties', 'description', '__proto__'])( + 'resolves the question named %s through schema properties', + (id) => { + const values = { + model: { value: 'jev-latest' }, + evaluationQuestions: { value: { [id]: questions.passed } }, + } + expect(getEffectiveBlockOutputPaths('agent', values)).toContain(`answers.${id}.noul`) + expect(getEffectiveBlockOutputType('agent', `answers.${id}.noul`, values)).toBe('number') + } + ) + }) + + it.each([false, true])( + 'serializes native fields without requiring messages, advanced=%s', + (advancedMode) => { + const values = { + model: 'jev-1.13.0', + apiKey: '{{TYPESAFE_API_KEY}}', + evaluationState: '42', + evaluationQuestions: '{"passed":{"type":"noul","instructions":"Did it pass?"}}', + messages: JSON.stringify([{ role: 'user', content: 'Old chat prompt' }]), + } + const block: BlockState = { + id: 'agent-test', + type: 'agent', + name: 'Evaluator', + position: { x: 0, y: 0 }, + enabled: true, + advancedMode, + outputs: {}, + subBlocks: Object.fromEntries( + Object.entries(values).map(([id, value]) => [ + id, + { id, value, type: AgentBlock.subBlocks.find((field) => field.id === id)!.type }, + ]) + ), + } + const result = new Serializer().serializeWorkflow({ [block.id]: block }, [], {}, {}, true) + expect(result.blocks[0].config.tool).toBe('typesafe') + expect(result.blocks[0].config.params).toMatchObject({ + evaluationState: '42', + evaluationQuestions: values.evaluationQuestions, + }) + expect(result.blocks[0].config.params).not.toHaveProperty('messages') + } + ) +}) diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 82464855edd..6cc01283268 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -6,8 +6,8 @@ import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model- import type { BlockConfig } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { + getAgentModelOptions, getModelCapabilityCondition, - getModelOptions, getProviderCredentialSubBlocks, getSerializedModelProviderId, normalizeFileInput, @@ -15,6 +15,7 @@ import { } from '@/blocks/utils' import { getBaseModelProviders, + getEvaluationModels, getMaxTemperature, getModelsWithDeepResearch, getModelsWithoutMemory, @@ -41,6 +42,8 @@ const MODELS_WITH_THINKING = getModelsWithThinking() const MODELS_WITH_PROMPT_CACHING = getModelsWithPromptCaching() const MODELS_WITH_DEEP_RESEARCH = getModelsWithDeepResearch() const MODELS_WITHOUT_MEMORY = getModelsWithoutMemory() +const EVALUATION_MODELS = getEvaluationModels() +const MODELS_WITHOUT_CHAT_CONTROLS = [...MODELS_WITH_DEEP_RESEARCH, ...EVALUATION_MODELS] interface AgentResponse extends ToolResponse { output: { @@ -82,7 +85,7 @@ export const AgentBlock: BlockConfig = { description: 'Build an agent', authMode: AuthMode.ApiKey, longDescription: - 'The Agent block is a core workflow block that is a wrapper around an LLM. It takes in system/user prompts and calls an LLM provider. It can also make tool calls by directly containing tools inside of its tool input. It can additionally return structured output.', + 'The Agent block is a core workflow block that is a wrapper around an LLM. It takes in system/user prompts and calls an LLM provider. It can also make tool calls by directly containing tools inside of its tool input. It can additionally return structured output. Select a Jev model to evaluate state against typed Choice, Score, and Noul questions and return structured answers.', bestPractices: ` - Prefer using integrations as tools within the agent block over separate integration blocks unless complete determinism needed. - Response Format should be a valid JSON Schema. This determines the output of the agent only if present. Fields can be accessed at root level by the following blocks: e.g. . If response format is not present, the agent will return the standard outputs: content, model, tokens, toolCalls. @@ -105,6 +108,7 @@ export const AgentBlock: BlockConfig = { subBlocks: [ { id: 'messages', + condition: { field: 'model', value: EVALUATION_MODELS, not: true }, title: 'Messages', type: 'messages-input', placeholder: 'Enter messages...', @@ -151,11 +155,41 @@ Return ONLY the JSON array.`, placeholder: 'Type or select a model...', required: true, defaultValue: 'claude-sonnet-5', - options: getModelOptions, + options: getAgentModelOptions, commandSearchable: true, }, + ...getProviderCredentialSubBlocks(), + { + id: 'evaluationState', + title: 'State', + type: 'long-input', + placeholder: 'Content or workflow data to evaluate...', + description: + 'Text, a JSON object, or an array shared by every question. Jev supports text only.', + required: true, + condition: getModelCapabilityCondition(EVALUATION_MODELS), + }, + { + id: 'evaluationQuestions', + title: 'Questions', + type: 'code', + language: 'json', + placeholder: '{"passed":{"type":"noul","instructions":"Did the task succeed?"}}', + description: + 'Questions keyed by ID: choice (1–255 named options), score (2–10 ordered levels), or noul (a yes/no probability). Answers use the same IDs.', + required: true, + condition: getModelCapabilityCondition(EVALUATION_MODELS), + wandConfig: { + enabled: true, + prompt: + 'Generate a JSON object of Jev evaluation questions keyed by descriptive IDs. Every question needs type and instructions. choice: criteria is an object with 1–255 option names mapped to descriptions or null. score: criteria is an array of 2–10 descriptions ordered lowest to highest. noul: criteria is optional, with true and false descriptions. Instructions and descriptions may be text, JSON objects, or arrays. Return only valid JSON. Current questions: {context}', + placeholder: 'Describe the decisions to make...', + generationType: 'json-object', + }, + }, { id: 'attachmentFiles', + condition: { field: 'model', value: EVALUATION_MODELS, not: true }, title: 'Files', type: 'file-upload', canonicalParamId: 'files', @@ -166,6 +200,7 @@ Return ONLY the JSON array.`, }, { id: 'files', + condition: { field: 'model', value: EVALUATION_MODELS, not: true }, title: 'Files', type: 'short-input', canonicalParamId: 'files', @@ -251,7 +286,6 @@ Return ONLY the JSON array.`, }, }, - ...getProviderCredentialSubBlocks(), { id: 'tools', title: 'Tools', @@ -259,7 +293,7 @@ Return ONLY the JSON array.`, defaultValue: [], condition: { field: 'model', - value: MODELS_WITH_DEEP_RESEARCH, + value: MODELS_WITHOUT_CHAT_CONTROLS, not: true, }, }, @@ -270,7 +304,7 @@ Return ONLY the JSON array.`, defaultValue: [], condition: { field: 'model', - value: MODELS_WITH_DEEP_RESEARCH, + value: MODELS_WITHOUT_CHAT_CONTROLS, not: true, }, }, @@ -406,7 +440,7 @@ Return ONLY the JSON array.`, mode: 'advanced', condition: { field: 'model', - value: MODELS_WITH_DEEP_RESEARCH, + value: MODELS_WITHOUT_CHAT_CONTROLS, not: true, }, }, @@ -418,7 +452,7 @@ Return ONLY the JSON array.`, language: 'json', condition: { field: 'model', - value: MODELS_WITH_DEEP_RESEARCH, + value: MODELS_WITHOUT_CHAT_CONTROLS, not: true, }, wandConfig: RESPONSE_FORMAT_WAND_CONFIG, @@ -433,7 +467,10 @@ Return ONLY the JSON array.`, value: MODELS_WITH_DEEP_RESEARCH, }, }, - getModelFallbackSubBlock(), + { + ...getModelFallbackSubBlock(), + condition: { field: 'model', value: EVALUATION_MODELS, not: true }, + }, ], tools: { access: [ @@ -501,6 +538,14 @@ Return ONLY the JSON array.`, }, }, inputs: { + evaluationState: { + type: 'string', + description: 'Content to evaluate: text, a JSON object, or an array', + }, + evaluationQuestions: { + type: 'json', + description: 'Map of question IDs to native Jev Choice, Score, or Noul questions', + }, messages: { type: 'json', description: @@ -601,6 +646,12 @@ Return ONLY the JSON array.`, skills: { type: 'json', description: 'Selected skills configuration' }, }, outputs: { + answers: { + type: 'json', + description: + 'Evaluation answers keyed by question ID: choice, score, or noul, with probabilities and confidence where applicable', + condition: { field: 'model', value: EVALUATION_MODELS, allowReference: true }, + }, content: { type: 'string', description: 'Generated response content' }, model: { type: 'string', description: 'Model used for generation' }, tokens: { type: 'json', description: 'Token usage statistics' }, diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index 92b43f7b483..34a1183d338 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -15,6 +15,7 @@ import { getProviderModels, isAutoModel, isCustomModelId, + isEvaluationModel, orderModelIdsByReleaseDate, SIM_AUTO_MODEL_ID, } from '@/providers/models' @@ -52,6 +53,15 @@ export const SERVICE_ACCOUNT_SUBBLOCKS: SubBlockConfig[] = [ * Returns model options for combobox subblocks, combining all provider sources. */ export function getModelOptions() { + return buildModelOptions(false) +} + +/** Agent supports both conversational and native evaluation models. */ +export function getAgentModelOptions() { + return buildModelOptions(true) +} + +function buildModelOptions(includeEvaluation: boolean) { const providersState = useProvidersStore.getState() const baseModels = orderModelIdsByReleaseDate(providersState.providers.base.models) const ollamaModels = providersState.providers.ollama.models @@ -77,7 +87,11 @@ export function getModelOptions() { ) const options = allModels - .filter((model) => getModelSunsetStatus(model) !== 'deprecated') + .filter( + (model) => + getModelSunsetStatus(model) !== 'deprecated' && + (includeEvaluation || !isEvaluationModel(model)) + ) .map((model) => { const icon = getProviderIcon(model) return { label: model, id: model, ...(icon && { icon }) } diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index 165cd584113..cf026bf4f5b 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -9936,3 +9936,16 @@ export function PitchBookIcon(props: SVGProps) { ) } + +/** TypeSafe’s official mark from https://typesafe.ai. */ +export function TypeSafeIcon(props: SVGProps) { + return ( + + + + ) +} diff --git a/apps/sim/ee/organization-usage/components/usage-consumers.tsx b/apps/sim/ee/organization-usage/components/usage-consumers.tsx index a4b9342bdba..8a508eaf2c9 100644 --- a/apps/sim/ee/organization-usage/components/usage-consumers.tsx +++ b/apps/sim/ee/organization-usage/components/usage-consumers.tsx @@ -23,6 +23,7 @@ import { OpenRouterIcon, SakanaIcon, TogetherIcon, + TypeSafeIcon, VertexIcon, VllmIcon, xAIIcon, @@ -74,6 +75,7 @@ const PROVIDER_ICONS: Readonly { resetDbChainMock() }) + describe('native evaluation models', () => { + const questions = { passed: { type: 'noul', instructions: 'Did the task succeed?' } } + const inputs: AgentInputs = { + model: 'jev-1.13.0', + apiKey: 'test-key', + evaluationState: 'Task complete', + evaluationQuestions: questions, + } + + it('uses the provider path with native inputs and exposes structured answers', async () => { + mockGetProviderFromModel.mockReturnValue('typesafe') + const answers = { passed: { type: 'noul', noul: 0.98 } } + mockExecuteProviderRequest.mockResolvedValue({ + content: JSON.stringify(answers), + answers, + model: inputs.model, + tokens: { input: 20, output: 5, total: 25 }, + }) + const result = await handler.execute(mockContext, mockBlock, inputs) + expect(mockValidateModelProvider).toHaveBeenCalledWith( + mockContext.userId, + mockContext.workspaceId, + inputs.model, + mockContext + ) + expect(mockExecuteProviderRequest).toHaveBeenCalledWith( + 'typesafe', + expect.objectContaining({ + model: inputs.model, + apiKey: 'test-key', + evaluation: { state: 'Task complete', questions }, + context: undefined, + systemPrompt: undefined, + tools: [], + }), + expect.anything() + ) + expect(result).toMatchObject({ + answers, + content: JSON.stringify(answers), + tokens: { total: 25 }, + }) + }) + + it('ignores saved chat settings after switching the model to Jev', async () => { + mockGetProviderFromModel.mockReturnValue('typesafe') + await handler.execute(mockContext, mockBlock, { + ...inputs, + messages: [{ role: 'user', content: 'Old conversation' }], + systemPrompt: 'Old prompt', + tools: [{ type: 'custom-tool', title: 'Stale tool' }], + skills: [{ skillId: 'stale-skill' }], + responseFormat: '{invalid stale JSON', + memoryType: 'conversation', + conversationId: 'stale-conversation', + temperature: 0.5, + maxTokens: 100, + files: [{ name: 'old.png' }], + fallbackModels: [{ model: 'gpt-4o' }], + }) + const request = mockExecuteProviderRequest.mock.calls[0][1] + expect(request).toMatchObject({ + evaluation: { state: 'Task complete', questions }, + tools: [], + context: undefined, + temperature: undefined, + maxTokens: undefined, + }) + expect(request.messages ?? []).toEqual([]) + expect(mockOpenAgentTurnSession).not.toHaveBeenCalled() + expect(mockExecuteProviderRequest).toHaveBeenCalledOnce() + }) + + it('removes saved evaluation inputs when switching back to a chat model', async () => { + await handler.execute(mockContext, mockBlock, { + ...inputs, + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello' }], + }) + expect(mockExecuteProviderRequest.mock.calls[0][1].evaluation).toBeUndefined() + }) + + it.each(['evaluationState', 'evaluationQuestions'] as const)( + 'projects secrets in %s before the provider boundary', + async (field) => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'PRIVATE_TEXT', plaintext: 'private value', encryptedValue: 'encrypted' }, + ]) + const path = field === 'evaluationState' ? [field] : [field, 'passed', 'instructions'] + registry.recordResolvedAtInputPath('PRIVATE_TEXT', 'private value', path) + registry.recordResolvedInputProjection(path, 'private value', '{{PRIVATE_TEXT}}') + mockContext.resolvedSecretTraceRegistry = registry + mockGetProviderFromModel.mockReturnValue('typesafe') + await handler.execute(mockContext, mockBlock, { + ...inputs, + [field]: + field === 'evaluationState' + ? 'private value' + : { passed: { type: 'noul', instructions: 'private value' } }, + }) + const request = mockExecuteProviderRequest.mock.calls[0][1] + expect(JSON.stringify(request.evaluation)).not.toContain('private value') + expect(JSON.stringify(request.evaluation)).toContain('{{PRIVATE_TEXT}}') + expect(request.apiKey).toBe('test-key') + } + ) + + it('refuses an evaluation model as a conversational fallback before executing the primary', async () => { + await expect( + handler.execute(mockContext, mockBlock, { + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello' }], + fallbackModels: [{ model: 'jev-latest' }], + }) + ).rejects.toThrow('Evaluation models cannot serve as chat fallbacks') + expect(mockExecuteProviderRequest).not.toHaveBeenCalled() + }) + }) + describe('canHandle', () => { it('should return true for blocks with metadata id "agent"', () => { expect(handler.canHandle(mockBlock)).toBe(true) diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 1d21c6ffd50..38fa9e1f054 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -120,7 +120,7 @@ import { canUseProviderLargeFilePath, getInlineHydrationMaxBytes, } from '@/providers/file-attachments.server' -import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' +import { isAutoModel, isEvaluationModel, SIM_AUTO_MODEL_ID } from '@/providers/models' import { type ProviderToolInputProvenance, registerProviderToolInputProvenance, @@ -321,6 +321,15 @@ export class AgentBlockHandler implements BlockHandler { inputs: AgentInputs, nodeMetadata?: BlockNodeMetadata ): Promise { + /** Inactive fields can remain saved when the builder switches model modalities. */ + inputs = isEvaluationModel(inputs.model || AGENT.DEFAULT_MODEL) + ? { + model: inputs.model, + apiKey: inputs.apiKey, + evaluationState: inputs.evaluationState, + evaluationQuestions: inputs.evaluationQuestions, + } + : inputs ctx.mcpBlockId = block.id const providerErrorRegistry = ctx.resolvedSecretTraceRegistry?.forkForInputPaths( AGENT_RAW_PROVIDER_ERROR_INPUT_PATHS @@ -379,6 +388,10 @@ export class AgentBlockHandler implements BlockHandler { userPrompt: filteredInputs.userPrompt, messages: filteredInputs.messages, memories: filteredInputs.memories, + ...(isEvaluationModel(filteredInputs.model || AGENT.DEFAULT_MODEL) && { + evaluationState: filteredInputs.evaluationState, + evaluationQuestions: filteredInputs.evaluationQuestions, + }), }, coreModelInputPaths ) @@ -552,6 +565,9 @@ export class AgentBlockHandler implements BlockHandler { filteredInputs.fallbackModels, logger ) + if (configuredFallbacks.some((candidate) => isEvaluationModel(candidate.model))) { + throw new Error('Evaluation models cannot serve as chat fallbacks') + } const retry = nodeMetadata?.retry const fallbacksHeld = retry !== undefined && !retry.isFinalTry const fallbackCandidates = @@ -2487,6 +2503,9 @@ export class AgentBlockHandler implements BlockHandler { private getModelInputPaths(inputs: AgentInputs): ResolvedSecretInputPath[] { const paths: ResolvedSecretInputPath[] = [['systemPrompt'], ['userPrompt']] + if (isEvaluationModel(inputs.model || AGENT.DEFAULT_MODEL)) { + paths.push(['evaluationState'], ['evaluationQuestions']) + } for (let index = 0; index < (inputs.messages?.length ?? 0); index++) { const message = inputs.messages?.[index] const messageRoot = ['messages', String(index)] as const @@ -2841,8 +2860,11 @@ export class AgentBlockHandler implements BlockHandler { return { provider: providerId, model, + evaluation: isEvaluationModel(model) + ? { state: inputs.evaluationState, questions: inputs.evaluationQuestions } + : undefined, systemPrompt: validMessages ? undefined : inputs.systemPrompt, - context: validMessages ? undefined : stringifyJSON(messages), + context: validMessages || isEvaluationModel(model) ? undefined : stringifyJSON(messages), tools: formattedTools, temperature: inputs.temperature != null && inputs.temperature !== '' @@ -2944,6 +2966,7 @@ export class AgentBlockHandler implements BlockHandler { providerId, { model, + evaluation: providerRequest.evaluation, systemPrompt: 'systemPrompt' in providerRequest ? providerRequest.systemPrompt : undefined, context: 'context' in providerRequest ? providerRequest.context : undefined, @@ -3273,6 +3296,7 @@ export class AgentBlockHandler implements BlockHandler { private processStandardResponse(result: any): BlockOutput { return { content: result.content, + ...(result.answers && { answers: result.answers }), ...this.createResponseMetadata(result), ...(result.interactionId && { interactionId: result.interactionId }), } diff --git a/apps/sim/executor/handlers/agent/types.ts b/apps/sim/executor/handlers/agent/types.ts index b5ce98a6c70..a092fee6959 100644 --- a/apps/sim/executor/handlers/agent/types.ts +++ b/apps/sim/executor/handlers/agent/types.ts @@ -15,6 +15,8 @@ export interface SkillInput { } export interface AgentInputs { + evaluationState?: unknown + evaluationQuestions?: unknown model?: string responseFormat?: string | object tools?: ToolInput[] diff --git a/apps/sim/lib/api-key/byok.test.ts b/apps/sim/lib/api-key/byok.test.ts index 5a905237cf2..3e0e7893afc 100644 --- a/apps/sim/lib/api-key/byok.test.ts +++ b/apps/sim/lib/api-key/byok.test.ts @@ -59,6 +59,7 @@ vi.mock('@/stores/providers/store', () => ({ useProvidersStore: { getState: vi.fn() }, })) +import { byokProviderIdSchema } from '@/lib/api/contracts/byok-keys' import { getApiKeyWithBYOK, getBYOKKey } from '@/lib/api-key/byok' import { useProvidersStore } from '@/stores/providers/store' @@ -544,6 +545,89 @@ describe('getApiKeyWithBYOK provider classification', () => { }) }) +describe('getApiKeyWithBYOK for TypeSafe', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockIsHosted.value = true + mockGetHostedModels.mockReturnValue(['jev-latest', 'jev-1.13.0', 'jev-preview']) + mockGetRotatingApiKey.mockReturnValue('hosted-typesafe-key') + mockDecryptSecret.mockImplementation(async (encrypted: string) => ({ + decrypted: encrypted.replace('encrypted-', 'decrypted-'), + })) + mockIsOrganizationBYOKEntitled.mockResolvedValue(true) + }) + + it('accepts TypeSafe in workspace and organization BYOK contracts', () => { + expect(byokProviderIdSchema.parse('typesafe')).toBe('typesafe') + }) + + it.each(['jev-latest', 'jev-1.13.0', 'jev-preview'])( + 'resolves the platform pool when %s has no BYOK key', + async (model) => { + await expect(getApiKeyWithBYOK('typesafe', model, uniqueWorkspaceId())).resolves.toEqual({ + apiKey: 'hosted-typesafe-key', + isBYOK: false, + }) + expect(mockGetRotatingApiKey).toHaveBeenCalledWith('typesafe') + } + ) + + it('prefers the workspace pool without selecting a hosted key', async () => { + dbChainMockFns.orderBy.mockResolvedValueOnce([storedKey('workspace-key')]) + await expect(getApiKeyWithBYOK('typesafe', 'jev-latest', uniqueWorkspaceId())).resolves.toEqual( + { + apiKey: 'decrypted-workspace-key', + isBYOK: true, + scope: 'workspace', + } + ) + expect(mockGetRotatingApiKey).not.toHaveBeenCalled() + expect(mockIsOrganizationBYOKEntitled).not.toHaveBeenCalled() + }) + + it('inherits an entitled organization pool before using hosted credits', async () => { + dbChainMockFns.orderBy + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([storedOrganizationKey(uniqueOrganizationId(), 'organization-key')]) + await expect(getApiKeyWithBYOK('typesafe', 'jev-latest', uniqueWorkspaceId())).resolves.toEqual( + { + apiKey: 'decrypted-organization-key', + isBYOK: true, + scope: 'organization', + } + ) + expect(mockGetRotatingApiKey).not.toHaveBeenCalled() + }) + + it('rejects missing hosted credentials instead of making an unauthenticated request', async () => { + mockGetRotatingApiKey.mockImplementation(() => { + throw new Error('No configured key') + }) + await expect(getApiKeyWithBYOK('typesafe', 'jev-latest', uniqueWorkspaceId())).rejects.toThrow( + 'No API key available for typesafe jev-latest' + ) + }) + + it('never gives the hosted key to an unlisted model', async () => { + await expect(getApiKeyWithBYOK('typesafe', 'jev-custom', uniqueWorkspaceId())).rejects.toThrow( + 'API key is required' + ) + expect(mockGetRotatingApiKey).not.toHaveBeenCalled() + }) + + it('requires caller credentials on self-hosted deployments', async () => { + mockIsHosted.value = false + await expect( + getApiKeyWithBYOK('typesafe', 'jev-latest', uniqueWorkspaceId(), 'caller-key') + ).resolves.toEqual({ apiKey: 'caller-key', isBYOK: false }) + await expect(getApiKeyWithBYOK('typesafe', 'jev-latest', uniqueWorkspaceId())).rejects.toThrow( + 'API key is required' + ) + expect(mockGetRotatingApiKey).not.toHaveBeenCalled() + }) +}) + describe('getApiKeyWithBYOK for Fireworks', () => { const HOSTED_POOL_MODEL = 'fireworks/glm-5.2' diff --git a/apps/sim/lib/api-key/byok.ts b/apps/sim/lib/api-key/byok.ts index a9ce7d43e08..3a5088f3c6c 100644 --- a/apps/sim/lib/api-key/byok.ts +++ b/apps/sim/lib/api-key/byok.ts @@ -328,6 +328,7 @@ export async function getApiKeyWithBYOK( const isZaiModel = provider === 'zai' const isXaiModel = provider === 'xai' const isKimiModel = provider === 'kimi' + const isTypeSafeModel = provider === 'typesafe' const byokProviderId = isGeminiModel ? 'google' : (provider as BYOKProviderId) @@ -340,7 +341,8 @@ export async function getApiKeyWithBYOK( isMistralModel || isZaiModel || isXaiModel || - isKimiModel) + isKimiModel || + isTypeSafeModel) ) { const hostedModels = getHostedModels() const isModelHosted = hostedModels.some((m) => m.toLowerCase() === model.toLowerCase()) diff --git a/apps/sim/lib/api/contracts/byok-keys.ts b/apps/sim/lib/api/contracts/byok-keys.ts index d6ee1be20d7..e0333635231 100644 --- a/apps/sim/lib/api/contracts/byok-keys.ts +++ b/apps/sim/lib/api/contracts/byok-keys.ts @@ -9,6 +9,7 @@ export const byokProviderIdSchema = z.enum([ 'mistral', 'zai', 'kimi', + 'typesafe', 'xai', 'fireworks', 'together', diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 7163ec25cea..9a21aba0699 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -261,6 +261,9 @@ export const env = createEnv({ KIMI_API_KEY_1: z.string().min(1).optional(), // Primary Kimi (Moonshot AI) API key for load balancing KIMI_API_KEY_2: z.string().min(1).optional(), // Additional Kimi API key for load balancing KIMI_API_KEY_3: z.string().min(1).optional(), // Additional Kimi API key for load balancing + TYPESAFE_API_KEY_1: z.string().min(1).optional(), + TYPESAFE_API_KEY_2: z.string().min(1).optional(), + TYPESAFE_API_KEY_3: z.string().min(1).optional(), XAI_API_KEY_1: z.string().min(1).optional(), // Primary xAI API key for load balancing XAI_API_KEY_2: z.string().min(1).optional(), // Additional xAI API key for load balancing XAI_API_KEY_3: z.string().min(1).optional(), // Additional xAI API key for load balancing diff --git a/apps/sim/lib/core/utils.test.ts b/apps/sim/lib/core/utils.test.ts index 1c882094c52..2919ac7dd13 100644 --- a/apps/sim/lib/core/utils.test.ts +++ b/apps/sim/lib/core/utils.test.ts @@ -24,6 +24,9 @@ beforeAll(() => { XAI_API_KEY_1: 'test-xai-key-1', XAI_API_KEY_2: 'test-xai-key-2', XAI_API_KEY_3: 'test-xai-key-3', + TYPESAFE_API_KEY_1: 'test-typesafe-key-1', + TYPESAFE_API_KEY_2: 'test-typesafe-key-2', + TYPESAFE_API_KEY_3: 'test-typesafe-key-3', FIREWORKS_API_KEY_1: 'test-fireworks-key-1', FIREWORKS_API_KEY_2: 'test-fireworks-key-2', FIREWORKS_API_KEY_3: 'test-fireworks-key-3', @@ -321,6 +324,9 @@ describe('getInvalidCharacters', () => { }) describe('getRotatingApiKey', () => { + it.concurrent('rotates the TypeSafe key pool through the shared selector', () => { + expect(getRotatingApiKey('typesafe')).toMatch(/^test-typesafe-key-[1-3]$/) + }) it.concurrent('should return OpenAI API key based on current minute', () => { const result = getRotatingApiKey('openai') expect(result).toMatch(/^test-openai-key-[1-3]$/) diff --git a/apps/sim/lib/memory/bounded-json.test.ts b/apps/sim/lib/core/utils/bounded-json.test.ts similarity index 79% rename from apps/sim/lib/memory/bounded-json.test.ts rename to apps/sim/lib/core/utils/bounded-json.test.ts index 52a5c584a10..3f5e503b57f 100644 --- a/apps/sim/lib/memory/bounded-json.test.ts +++ b/apps/sim/lib/core/utils/bounded-json.test.ts @@ -1,8 +1,8 @@ /** @vitest-environment node */ import { describe, expect, it, vi } from 'vitest' -import { stringifyBoundedMemoryJson } from '@/lib/memory/bounded-json' +import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' -describe('bounded memory JSON', () => { +describe('bounded JSON', () => { it.each([ { value: { text: 'hello', values: [1, false, null] } }, { value: { text: 'é😀\ud800\udc00\ud800' } }, @@ -12,8 +12,8 @@ describe('bounded memory JSON', () => { ])('uses the caller byte limit including UTF-8 and escaped JSON bytes', ({ value }) => { const json = JSON.stringify(value) const bytes = Buffer.byteLength(json, 'utf8') - expect(stringifyBoundedMemoryJson(value, bytes)).toBe(json) - expect(stringifyBoundedMemoryJson(value, bytes - 1)).toBeUndefined() + expect(stringifyBoundedJson(value, bytes)).toBe(json) + expect(stringifyBoundedJson(value, bytes - 1)).toBeUndefined() }) it('rejects cycles, excessive depth, and excessive nodes', () => { @@ -27,7 +27,7 @@ describe('bounded memory JSON', () => { Array(100_001), Object.fromEntries(Array.from({ length: 100_001 }, (_, index) => [index, undefined])), ]) { - expect(stringifyBoundedMemoryJson(value, 8 * 1024 * 1024)).toBeUndefined() + expect(stringifyBoundedJson(value, 8 * 1024 * 1024)).toBeUndefined() } }) @@ -37,7 +37,7 @@ describe('bounded memory JSON', () => { const accessor = Object.defineProperty({}, 'secret', { enumerable: true, get: getter }) const custom = Object.defineProperty({}, 'toJSON', { value: toJSON }) for (const value of [accessor, custom, { output: new Uint8Array([1, 2, 3]) }]) { - expect(stringifyBoundedMemoryJson(value, 1024)).toBeUndefined() + expect(stringifyBoundedJson(value, 1024)).toBeUndefined() } expect(getter).not.toHaveBeenCalled() expect(toJSON).not.toHaveBeenCalled() @@ -47,7 +47,7 @@ describe('bounded memory JSON', () => { const value = { output: 'x'.repeat(1025) } const serialize = vi.spyOn(JSON, 'stringify') try { - expect(stringifyBoundedMemoryJson(value, 1024)).toBeUndefined() + expect(stringifyBoundedJson(value, 1024)).toBeUndefined() expect(serialize).not.toHaveBeenCalled() } finally { serialize.mockRestore() @@ -61,7 +61,7 @@ describe('bounded memory JSON', () => { ])('rejects escaped bytes before serializing the captured graph', ({ value }) => { const serialize = vi.spyOn(JSON, 'stringify') try { - expect(stringifyBoundedMemoryJson(value, 1024)).toBeUndefined() + expect(stringifyBoundedJson(value, 1024)).toBeUndefined() expect(serialize).not.toHaveBeenCalled() } finally { serialize.mockRestore() @@ -71,7 +71,7 @@ describe('bounded memory JSON', () => { it('serializes the admitted descriptors without reading proxy values or toJSON', () => { const get = vi.fn(() => 'UNADMITTED') const value = new Proxy({ text: 'admitted' }, { get }) - expect(stringifyBoundedMemoryJson(value, 1024)).toBe('{"text":"admitted"}') + expect(stringifyBoundedJson(value, 1024)).toBe('{"text":"admitted"}') expect(get).not.toHaveBeenCalled() }) @@ -79,13 +79,13 @@ describe('bounded memory JSON', () => { const get = vi.fn(() => 'UNADMITTED') const prototype = Object.create(Array.prototype, { 0: { get } }) const value = Object.setPrototypeOf(Array(1), prototype) - expect(stringifyBoundedMemoryJson(value, 1024)).toBe('[null]') + expect(stringifyBoundedJson(value, 1024)).toBe('[null]') expect(get).not.toHaveBeenCalled() }) it('allows repeated references without treating them as a cycle', () => { const result = { answer: 42 } const value = { rawResponse: result, modelResponse: result } - expect(stringifyBoundedMemoryJson(value, 1024)).toBe(JSON.stringify(value)) + expect(stringifyBoundedJson(value, 1024)).toBe(JSON.stringify(value)) }) }) diff --git a/apps/sim/lib/memory/bounded-json.ts b/apps/sim/lib/core/utils/bounded-json.ts similarity index 89% rename from apps/sim/lib/memory/bounded-json.ts rename to apps/sim/lib/core/utils/bounded-json.ts index 698d1afbbe5..55678f5cb3a 100644 --- a/apps/sim/lib/memory/bounded-json.ts +++ b/apps/sim/lib/core/utils/bounded-json.ts @@ -1,5 +1,5 @@ -const MAX_MEMORY_JSON_NODES = 100_000 -const MAX_MEMORY_JSON_DEPTH = 64 +const MAX_JSON_NODES = 100_000 +const MAX_JSON_DEPTH = 64 /** Counts JSON escapes without allocating the escaped string. */ function quotedStringBytes(value: string, remaining: number): number | undefined { @@ -22,7 +22,7 @@ function quotedStringBytes(value: string, remaining: number): number | undefined } /** Captures bounded plain JSON once, without executing accessors or serializing the source graph. */ -export function stringifyBoundedMemoryJson(value: unknown, maxBytes: number): string | undefined { +export function stringifyBoundedJson(value: unknown, maxBytes: number): string | undefined { let nodes = 0 let bytes = 0 const invalid = Symbol('invalid JSON') @@ -32,7 +32,7 @@ export function stringifyBoundedMemoryJson(value: unknown, maxBytes: number): st return bytes <= maxBytes } const capture = (item: unknown, depth: number): unknown => { - if (++nodes > MAX_MEMORY_JSON_NODES || depth > MAX_MEMORY_JSON_DEPTH) return invalid + if (++nodes > MAX_JSON_NODES || depth > MAX_JSON_DEPTH) return invalid if (typeof item === 'string') { const count = quotedStringBytes(item, maxBytes - bytes) if (count === undefined || !addBytes(count)) return invalid @@ -53,7 +53,7 @@ export function stringifyBoundedMemoryJson(value: unknown, maxBytes: number): st : Object.create(null) if (isArray) { const length = Object.getOwnPropertyDescriptor(item, 'length')?.value - if (typeof length !== 'number' || length > MAX_MEMORY_JSON_NODES - nodes) return invalid + if (typeof length !== 'number' || length > MAX_JSON_NODES - nodes) return invalid for (let index = 0; index < length; index++) { const field = Object.getOwnPropertyDescriptor(item, index) if (field && !('value' in field)) return invalid @@ -69,7 +69,7 @@ export function stringifyBoundedMemoryJson(value: unknown, maxBytes: number): st if (!field || !field.enumerable) continue if (!('value' in field)) return invalid if (field.value === undefined) { - if (++nodes > MAX_MEMORY_JSON_NODES) return invalid + if (++nodes > MAX_JSON_NODES) return invalid continue } const keyBytes = quotedStringBytes(key, maxBytes - bytes) diff --git a/apps/sim/lib/memory/agent-turn-session.ts b/apps/sim/lib/memory/agent-turn-session.ts index 090fe06f792..eef5d6d6750 100644 --- a/apps/sim/lib/memory/agent-turn-session.ts +++ b/apps/sim/lib/memory/agent-turn-session.ts @@ -6,6 +6,7 @@ import { isRecordLike } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { decryptSecret } from '@/lib/core/security/encryption' +import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' import { bindDurableSecretProvenanceToValue, durableSecretProvenanceFromRegistry, @@ -24,7 +25,6 @@ import { } from '@/lib/memory/application/agent-turns' import { MEMORY_DELEGATION_AUDIENCE } from '@/lib/memory/application/authorization' import { getMemoryArtifactHandle } from '@/lib/memory/artifact-handle' -import { stringifyBoundedMemoryJson } from '@/lib/memory/bounded-json' import { decryptMemoryCheckpoint, encryptMemoryCheckpoint, @@ -534,7 +534,7 @@ export async function openAgentTurnSession( }, async prepareResult(result) { let requiresArtifact = - stringifyBoundedMemoryJson(result, MEMORY.MAX_MESSAGE_CONTENT_BYTES) === undefined + stringifyBoundedJson(result, MEMORY.MAX_MESSAGE_CONTENT_BYTES) === undefined let safeError: string | undefined try { const projected = await project(result.modelResponse) @@ -550,7 +550,7 @@ export async function openAgentTurnSession( modelResponse: { ...result.modelResponse, ...projected }, } requiresArtifact ||= - stringifyBoundedMemoryJson(prepared, MEMORY.MAX_MESSAGE_CONTENT_BYTES) === undefined + stringifyBoundedJson(prepared, MEMORY.MAX_MESSAGE_CONTENT_BYTES) === undefined requiresArtifact ||= JSON.stringify(prepared.modelResponse).length > MAX_ARTIFACT_PREVIEW_CHARS if (requiresArtifact) { diff --git a/apps/sim/lib/memory/artifacts.ts b/apps/sim/lib/memory/artifacts.ts index b6d757bb708..8f575dd9f00 100644 --- a/apps/sim/lib/memory/artifacts.ts +++ b/apps/sim/lib/memory/artifacts.ts @@ -2,13 +2,13 @@ import { dbFor } from '@sim/db' import { executionLargeValues, memory, memoryArtifact } from '@sim/db/schema' import { and, eq, isNull, sql } from 'drizzle-orm' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' +import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' import { collectLargeValueReferenceKeys, registerLargeValueOwner, } from '@/lib/execution/payloads/large-value-metadata' import { isLargeValueRef, type LargeValueRef } from '@/lib/execution/payloads/large-value-ref' import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store' -import { stringifyBoundedMemoryJson } from '@/lib/memory/bounded-json' export const MAX_MEMORY_ARTIFACT_BYTES = 8 * 1024 * 1024 export const MAX_MEMORY_ARTIFACT_STORED_BYTES = MAX_MEMORY_ARTIFACT_BYTES * 2 + 1024 @@ -71,7 +71,7 @@ function activeMemoryPredicate(scope: MemoryArtifactScope) { export async function storeMemoryArtifact( input: StoreMemoryArtifactInput ): Promise { - const json = stringifyBoundedMemoryJson(input.value, MAX_MEMORY_ARTIFACT_BYTES) + const json = stringifyBoundedJson(input.value, MAX_MEMORY_ARTIFACT_BYTES) if (json === undefined) return undefined const execDb = dbFor('exec') const [conversation] = await execDb @@ -183,9 +183,7 @@ export async function readMemoryArtifact(input: ReadMemoryArtifactInput): Promis const { decrypted } = await decryptSecret(envelope.encrypted, { logFailure: false }) if (Buffer.byteLength(decrypted, 'utf8') > MAX_MEMORY_ARTIFACT_BYTES) return undefined const value: unknown = JSON.parse(decrypted) - return stringifyBoundedMemoryJson(value, MAX_MEMORY_ARTIFACT_BYTES) === undefined - ? undefined - : value + return stringifyBoundedJson(value, MAX_MEMORY_ARTIFACT_BYTES) === undefined ? undefined : value } catch { return undefined } diff --git a/apps/sim/lib/memory/conversation-store.ts b/apps/sim/lib/memory/conversation-store.ts index cfea3e9d2a3..1a2c50330f1 100644 --- a/apps/sim/lib/memory/conversation-store.ts +++ b/apps/sim/lib/memory/conversation-store.ts @@ -3,6 +3,7 @@ import { agentMemoryTurn, memory, memoryItem, memorySecretProvenance } from '@si import { generateId } from '@sim/utils/id' import { and, asc, desc, eq, gt, inArray, isNull, lt, type SQLWrapper, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' import type { DbOrTx, DbTransaction } from '@/lib/db/types' import { type DurableSecretProvenance, @@ -10,7 +11,6 @@ import { hashDurableSecretProvenanceValue, mergeDurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' -import { stringifyBoundedMemoryJson } from '@/lib/memory/bounded-json' import { lockMemoryConversationInTx } from '@/lib/memory/locks' import { MAX_RICH_MEMORY_PAGE_BYTES, PlainMemoryReadBudget } from '@/lib/memory/read-budget' import { @@ -98,7 +98,7 @@ interface PlainMemoryWriteInput { /** Both storage versions persist the same admitted snapshot of each new history item. */ function captureMemoryItem(value: unknown): unknown { - const encoded = stringifyBoundedMemoryJson(value, MAX_MEMORY_ITEM_BYTES) + const encoded = stringifyBoundedJson(value, MAX_MEMORY_ITEM_BYTES) if (encoded === undefined) throw new OrchestrationError( 'payload_too_large', diff --git a/apps/sim/lib/memory/retrieval-prefix.ts b/apps/sim/lib/memory/retrieval-prefix.ts index 0a1ed888efe..3a9f66096ff 100644 --- a/apps/sim/lib/memory/retrieval-prefix.ts +++ b/apps/sim/lib/memory/retrieval-prefix.ts @@ -1,9 +1,9 @@ import { dbFor } from '@sim/db' import { memory, memorySecretProvenance } from '@sim/db/schema' import { and, eq, isNull, sql } from 'drizzle-orm' +import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import type { MemoryArtifactScope } from '@/lib/memory/artifacts' -import { stringifyBoundedMemoryJson } from '@/lib/memory/bounded-json' import { readBoundMemorySecretProvenance } from '@/lib/memory/secret-provenance' export const MAX_MEMORY_RETRIEVAL_PREFIX_BYTES = 1024 * 1024 @@ -45,7 +45,7 @@ export async function readMemoryRetrievalPrefix( return { status: 'oversized' } if ( !Array.isArray(row.data) || - stringifyBoundedMemoryJson(row.data, MAX_MEMORY_RETRIEVAL_PREFIX_BYTES) === undefined + stringifyBoundedJson(row.data, MAX_MEMORY_RETRIEVAL_PREFIX_BYTES) === undefined ) return { status: 'unavailable' } const provenance = readBoundMemorySecretProvenance(row) diff --git a/apps/sim/lib/memory/retrieval.ts b/apps/sim/lib/memory/retrieval.ts index 8b369e06a9b..f2805b41e05 100644 --- a/apps/sim/lib/memory/retrieval.ts +++ b/apps/sim/lib/memory/retrieval.ts @@ -3,6 +3,7 @@ import { isRecordLike } from '@sim/utils/object' import { escapeRegExp } from '@sim/utils/string' import { z } from 'zod' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' import { type DurableSecretProvenance, EXACT_EMPTY_DURABLE_SECRET_PROVENANCE, @@ -17,7 +18,6 @@ import { type MemoryArtifactScope, readMemoryArtifactByHandle, } from '@/lib/memory/artifacts' -import { stringifyBoundedMemoryJson } from '@/lib/memory/bounded-json' import { readConversationItems } from '@/lib/memory/conversation-store' import { readMemoryRetrievalPrefix } from '@/lib/memory/retrieval-prefix' import type { ExecutionContext } from '@/executor/types' @@ -187,7 +187,7 @@ async function projectText( provenance: DurableSecretProvenance, provenanceValue: unknown ): Promise { - if (stringifyBoundedMemoryJson(value, MAX_MEMORY_ARTIFACT_BYTES) === undefined) return undefined + if (stringifyBoundedJson(value, MAX_MEMORY_ARTIFACT_BYTES) === undefined) return undefined const current = input.projection.resolvedSecretTraceRegistry const registry = current?.forkForToolCall() ?? new ResolvedSecretTraceRegistry([]) if (!(await importDurableSecretProvenance(registry, provenance, provenanceValue))) @@ -198,7 +198,7 @@ async function projectText( const safe = redaction?.enabled ? await redactObjectStrings(projected.value, { ...redaction, onFailure: 'throw' }) : projected.value - return stringifyBoundedMemoryJson(withOpaqueHandles(safe), MAX_MEMORY_ARTIFACT_BYTES) + return stringifyBoundedJson(withOpaqueHandles(safe), MAX_MEMORY_ARTIFACT_BYTES) } function textChunk(text: string, offset: number, args: MemoryRetrievalArguments) { diff --git a/apps/sim/lib/workflows/blocks/block-outputs.ts b/apps/sim/lib/workflows/blocks/block-outputs.ts index ed8f324b9e2..0a2aa28d3f4 100644 --- a/apps/sim/lib/workflows/blocks/block-outputs.ts +++ b/apps/sim/lib/workflows/blocks/block-outputs.ts @@ -3,7 +3,9 @@ import { extractFieldsFromSchema, parseResponseFormatSafely, } from '@/lib/core/utils/response-format' +import { getJevAnswerOutput } from '@/lib/workflows/blocks/jev-outputs' import { normalizeInputFormatValue } from '@/lib/workflows/input-format' +import { containsReference } from '@/lib/workflows/sanitization/references' import { classifyStartBlockType, StartBlockPath, @@ -23,6 +25,7 @@ import { type OutputFieldDefinition, } from '@/blocks/types' import { isHumanInTheLoopBlock } from '@/executor/constants' +import { isEvaluationModel } from '@/providers/models' import { getToolOutputsMetadata } from '@/tools/metadata-outputs' import { getTrigger, isTriggerValid } from '@/triggers' @@ -61,15 +64,19 @@ function evaluateOutputCondition( const fieldValue = subBlocks[condition.field]?.value + const deferred = + condition.allowReference && typeof fieldValue === 'string' && containsReference(fieldValue) let matches: boolean - if (Array.isArray(condition.value)) { + if (deferred) { + matches = true + } else if (Array.isArray(condition.value)) { // For array conditions, check if fieldValue is a valid primitive and included matches = isConditionPrimitive(fieldValue) && condition.value.includes(fieldValue) } else { matches = fieldValue === condition.value } - if (condition.not) { + if (condition.not && !deferred) { matches = !matches } @@ -428,6 +435,18 @@ export function getEffectiveBlockOutputs( const includeHidden = options?.includeHidden ?? false if (blockType === 'agent') { + const model = subBlocks?.model?.value + const mayEvaluate = + typeof model === 'string' && (isEvaluationModel(model) || containsReference(model)) + if (mayEvaluate) { + const outputs = getBlockOutputs('agent', subBlocks, false, { includeHidden }) + const answers = getJevAnswerOutput(subBlocks?.evaluationQuestions?.value) + return { + ...outputs, + ...(containsReference(model) ? getResponseFormatOutputs(subBlocks, 'agent') : undefined), + ...(answers ? { answers } : undefined), + } + } const responseFormatOutputs = getResponseFormatOutputs(subBlocks, 'agent') if (responseFormatOutputs) return responseFormatOutputs } @@ -542,9 +561,7 @@ function traverseOutputPath(outputs: OutputDefinition, pathParts: string[]): unk const currentObj = current as Record - if (part in currentObj) { - current = currentObj[part] - } else if ( + if ( 'type' in currentObj && (currentObj.type === 'object' || currentObj.type === 'json') && 'properties' in currentObj && @@ -575,6 +592,8 @@ function traverseOutputPath(outputs: OutputDefinition, pathParts: string[]): unk } else { return null } + } else if (part in currentObj) { + current = currentObj[part] } else { return null } diff --git a/apps/sim/lib/workflows/blocks/jev-outputs.ts b/apps/sim/lib/workflows/blocks/jev-outputs.ts new file mode 100644 index 00000000000..e2545e4bda1 --- /dev/null +++ b/apps/sim/lib/workflows/blocks/jev-outputs.ts @@ -0,0 +1,52 @@ +import { isRecordLike } from '@sim/utils/object' +import type { JevAnswer } from '@/providers/typesafe/types' +import type { OutputProperty } from '@/tools/types' + +const ANSWER_FIELDS = { + choice: { + choice: { type: 'string' }, + confidence: { type: 'number' }, + probabilities: { type: 'json' }, + type: { type: 'string' }, + }, + score: { + score: { type: 'number' }, + confidence: { type: 'number' }, + probabilities: { type: 'json' }, + legend: { type: 'json' }, + type: { type: 'string' }, + }, + noul: { + noul: { type: 'number' }, + type: { type: 'string' }, + }, +} satisfies { + [Type in JevAnswer['type']]: Record, OutputProperty> +} + +/** + * Describes answers known from the editor's questions without requiring resolved inputs. + * Keys containing reference syntax remain accessible through the whole answers object. + */ +export function getJevAnswerOutput( + questions: unknown +): (OutputProperty & { type: 'json' }) | undefined { + if (typeof questions === 'string') { + try { + questions = JSON.parse(questions) + } catch { + return undefined + } + } + if (!isRecordLike(questions)) return undefined + + const entries: Array<[string, OutputProperty]> = [] + for (const [id, question] of Object.entries(questions)) { + if (!/^[\w-]+$/.test(id) || !isRecordLike(question)) continue + const { type } = question + if (type !== 'choice' && type !== 'score' && type !== 'noul') continue + entries.push([id, { type: 'json', properties: ANSWER_FIELDS[type] }]) + } + + return entries.length > 0 ? { type: 'json', properties: Object.fromEntries(entries) } : undefined +} diff --git a/apps/sim/providers/conversation-continuation.ts b/apps/sim/providers/conversation-continuation.ts index 92941e2092b..84e5ccb3135 100644 --- a/apps/sim/providers/conversation-continuation.ts +++ b/apps/sim/providers/conversation-continuation.ts @@ -140,6 +140,7 @@ export async function restoreConversationNativeMessages( isChatCompletionsEndpoint(request?.azureEndpoint || env.AZURE_OPENAI_ENDPOINT || '') ? 'chat-completions' : providerHistoryProtocols[providerId] + if (!protocol) throw new Error('Evaluation providers do not support conversation history') const restored: Message[] = [] for (const group of groupConversationMessages(messages)) { const first = group[0] diff --git a/apps/sim/providers/history-adapters.test.ts b/apps/sim/providers/history-adapters.test.ts index ec2ee7785b0..55494a1a61b 100644 --- a/apps/sim/providers/history-adapters.test.ts +++ b/apps/sim/providers/history-adapters.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest' import type { ConversationProtocol } from '@/lib/memory/conversation-types' import { providerHistoryAdapters, providerHistoryProtocols } from '@/providers/history-adapters' -import { PROVIDER_DEFINITIONS } from '@/providers/models' +import { isEvaluationModel, PROVIDER_DEFINITIONS } from '@/providers/models' const fixtures: Array<{ protocol: ConversationProtocol; value: unknown }> = [ { @@ -121,7 +121,14 @@ describe('canonical provider wire adapters', () => { expect(Object.keys(providerHistoryProtocols).sort()).toEqual( Object.keys(PROVIDER_DEFINITIONS).sort() ) - for (const protocol of Object.values(providerHistoryProtocols)) - expect(providerHistoryAdapters[protocol]).toBeDefined() + for (const [providerId, protocol] of Object.entries(providerHistoryProtocols)) { + if (protocol === null) { + expect( + PROVIDER_DEFINITIONS[providerId].models.every((model) => isEvaluationModel(model.id)) + ).toBe(true) + } else { + expect(providerHistoryAdapters[protocol]).toBeDefined() + } + } }) }) diff --git a/apps/sim/providers/history-adapters.ts b/apps/sim/providers/history-adapters.ts index 7c629841129..07c1e70a341 100644 --- a/apps/sim/providers/history-adapters.ts +++ b/apps/sim/providers/history-adapters.ts @@ -80,7 +80,7 @@ export const providerHistoryAdapters: Record capture('bedrock', value) }, } -export const providerHistoryProtocols: Record = { +export const providerHistoryProtocols: Record = { openai: 'responses', 'azure-openai': 'responses', anthropic: 'anthropic', @@ -93,6 +93,7 @@ export const providerHistoryProtocols: Record cerebras: 'chat-completions', groq: 'chat-completions', sakana: 'chat-completions', + typesafe: null, nvidia: 'chat-completions', meta: 'chat-completions', zai: 'chat-completions', diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 9fa071ed27e..1129732b7e5 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -2078,3 +2078,112 @@ describe('executeProviderRequest — model level normalization', () => { expect(sentRequest().reasoningEffort).toBeUndefined() }) }) + +describe('native evaluation provider boundary', () => { + beforeEach(() => { + vi.clearAllMocks() + envFlagsMockFns.getCostMultiplier.mockReturnValue(2) + mockExecuteRequest.mockResolvedValue({ + content: '{"passed":true}', + model: 'jev-1.13.0', + answers: { passed: true }, + tokens: { input: 100, output: 10, total: 110 }, + }) + mockGetApiKeyWithBYOK.mockResolvedValue({ apiKey: 'resolved-typesafe-key', isBYOK: true }) + }) + + it.each([ + ['typesafe', { model: 'jev-1.13.0', messages: [{ role: 'user', content: 'Chat' }] }], + ['openai', { model: 'gpt-4o', evaluation: { state: 'Test', questions: {} } }], + ] satisfies Array<[string, ProviderRequest]>)( + 'rejects a mismatched %s request modality', + async (provider, request) => { + await expect(executeProviderRequest(provider, request)).rejects.toThrow( + 'same evaluation or chat modality' + ) + expect(mockExecuteRequest).not.toHaveBeenCalled() + } + ) + + it('resolves BYOK credentials and keeps evaluation answers without charging Sim credits', async () => { + const evaluation = { + state: 'Task complete', + questions: { passed: { type: 'noul', instructions: 'Passed?' } }, + } + const result = await executeProviderRequest('typesafe', { + model: 'jev-1.13.0', + apiKey: 'test-key', + workspaceId: 'test-workspace', + evaluation, + }) + expect(mockGetApiKeyWithBYOK).toHaveBeenCalledWith( + 'typesafe', + 'jev-1.13.0', + 'test-workspace', + 'test-key' + ) + expect(mockExecuteRequest).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: 'resolved-typesafe-key', evaluation }) + ) + expect(result).toMatchObject({ + answers: { passed: true }, + tokens: { total: 110 }, + cost: { input: 0, output: 0, total: 0 }, + }) + }) + + it.each(['jev-latest', 'jev-1.13.0', 'jev-preview'])( + 'bills hosted %s using the resolved model price and shared multiplier once', + async (model) => { + mockGetApiKeyWithBYOK.mockResolvedValue({ apiKey: 'hosted-typesafe-key', isBYOK: false }) + const result = await executeProviderRequest('typesafe', { + model, + workspaceId: 'test-workspace', + evaluation: { + state: 'Task complete', + questions: { passed: { type: 'noul', instructions: 'Passed?' } }, + }, + }) + expect(mockExecuteRequest).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: 'hosted-typesafe-key', isBYOK: false }) + ) + expect(result).toMatchObject({ + model: 'jev-1.13.0', + cost: { input: 0.0000084, output: 0, total: 0.0000084 }, + }) + } + ) + + it.each([false, true])('applies Jev streaming billing consistently, BYOK=%s', async (isBYOK) => { + mockGetApiKeyWithBYOK.mockResolvedValue({ apiKey: 'resolved-typesafe-key', isBYOK }) + const streaming: StreamingExecution = { + stream: new ReadableStream(), + execution: { + success: true, + output: { + content: '{"passed":{"type":"noul","noul":0.9}}', + answers: { passed: { type: 'noul', noul: 0.9 } }, + model: 'jev-1.13.0', + tokens: { input: 100, output: 10, total: 110 }, + cost: { input: 0.0000042, output: 0, total: 0.0000042 }, + }, + logs: [], + }, + } + mockExecuteRequest.mockResolvedValue(streaming) + await executeProviderRequest('typesafe', { + model: 'jev-latest', + workspaceId: 'test-workspace', + stream: true, + evaluation: { + state: 'Task complete', + questions: { passed: { type: 'noul', instructions: 'Passed?' } }, + }, + }) + expect(streaming.execution.output.cost).toMatchObject({ + input: isBYOK ? 0 : 0.0000084, + output: 0, + total: isBYOK ? 0 : 0.0000084, + }) + }) +}) diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 19760cd7a01..c616100d449 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -35,7 +35,7 @@ import { attachLargeFileRemoteUrls, uploadLargeFilesToProvider, } from '@/providers/file-attachments.server' -import { isKnownModelId } from '@/providers/models' +import { isEvaluationModel, isKnownModelId } from '@/providers/models' import { getProviderExecutor } from '@/providers/registry' import { type ProviderRuntimeContext, @@ -251,6 +251,10 @@ export async function executeProviderRequest( throw new Error(`Provider ${providerId} does not implement executeRequest`) } + if (isEvaluationModel(request.model) !== Boolean(request.evaluation)) { + throw new Error('The selected model and request must use the same evaluation or chat modality') + } + let resolvedRequest = sanitizeRequest(request) let isBYOK = false diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index 49fc8ac91c5..323012c690c 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -28,6 +28,7 @@ import { OpenRouterIcon, SakanaIcon, TogetherIcon, + TypeSafeIcon, VertexIcon, VllmIcon, xAIIcon, @@ -83,6 +84,8 @@ export interface ModelCapabilities { */ streamed?: ThinkingStreamVisibility } + /** Uses native state and questions instead of a conversational prompt. */ + evaluation?: boolean deepResearch?: boolean /** Whether this model supports conversation memory. Defaults to true if omitted. */ memory?: boolean @@ -167,6 +170,34 @@ export function getProviderFileAttachment(providerId: string): ProviderFileAttac } export const PROVIDER_DEFINITIONS: Record = { + typesafe: { + id: 'typesafe', + name: 'TypeSafe', + description: 'Jev evaluation models for classification, scoring, and agent decisions', + icon: TypeSafeIcon, + color: '#F386A1', + models: [ + { + id: 'jev-latest', + pricing: { input: 0.042, output: 0, updatedAt: '2026-09-22' }, + capabilities: { evaluation: true, memory: false }, + contextWindow: 64000, + }, + { + id: 'jev-1.13.0', + pricing: { input: 0.042, output: 0, updatedAt: '2026-09-22' }, + capabilities: { evaluation: true, memory: false }, + contextWindow: 64000, + }, + { + id: 'jev-preview', + pricing: { input: 0.042, output: 0, updatedAt: '2026-09-22' }, + capabilities: { evaluation: true, memory: false }, + contextWindow: 64000, + }, + ], + defaultModel: 'jev-latest', + }, fireworks: { id: 'fireworks', name: 'Fireworks', @@ -5392,6 +5423,7 @@ export function getProvidersWithToolUsageControl(): string[] { export function getHostedModels(): string[] { return [ + ...getProviderModels('typesafe'), ...getProviderModels('openai'), ...getProviderModels('anthropic'), ...getProviderModels('google'), @@ -5825,6 +5857,17 @@ export function getThinkingStreamVisibility(modelId: string): ThinkingStreamVisi return null } +/** Models that consume native evaluation inputs in the Agent block. */ +export function getEvaluationModels(): string[] { + return Object.values(PROVIDER_DEFINITIONS).flatMap((provider) => + provider.models.filter((model) => model.capabilities.evaluation).map((model) => model.id) + ) +} + +export function isEvaluationModel(modelId: string): boolean { + return getModelCapabilities(modelId)?.evaluation === true +} + /** * Get all models that support deep research capability */ diff --git a/apps/sim/providers/registry.ts b/apps/sim/providers/registry.ts index a1d70caa046..0088ac0f2ac 100644 --- a/apps/sim/providers/registry.ts +++ b/apps/sim/providers/registry.ts @@ -21,6 +21,7 @@ import { openRouterProvider } from '@/providers/openrouter' import { sakanaProvider } from '@/providers/sakana' import { togetherProvider } from '@/providers/together' import type { ProviderConfig, ProviderId } from '@/providers/types' +import { typesafeProvider } from '@/providers/typesafe' import { vertexProvider } from '@/providers/vertex' import { vllmProvider } from '@/providers/vllm' import { xAIProvider } from '@/providers/xai' @@ -39,6 +40,7 @@ const providerRegistry: Record = { cerebras: cerebrasProvider, groq: groqProvider, sakana: sakanaProvider, + typesafe: typesafeProvider, nvidia: nvidiaProvider, meta: metaProvider, zai: zaiProvider, diff --git a/apps/sim/providers/types.ts b/apps/sim/providers/types.ts index 43d8898058c..2cda6e7377c 100644 --- a/apps/sim/providers/types.ts +++ b/apps/sim/providers/types.ts @@ -14,6 +14,7 @@ export type ProviderId = | 'cerebras' | 'groq' | 'sakana' + | 'typesafe' | 'nvidia' | 'meta' | 'zai' @@ -93,6 +94,8 @@ export type TimeSegment = ProviderTimingSegment export interface ProviderResponse { content: string + /** Structured answers returned by a native evaluation model. */ + answers?: Record model: string tokens?: { /** Tokens billed at the base input rate, excluding cache reads and writes. */ @@ -191,7 +194,14 @@ export interface Message { tool_call_id?: string } +/** Native evaluation values are validated against the selected provider's schema. */ +export interface EvaluationInput { + state: unknown + questions: unknown +} + export interface ProviderRequest { + evaluation?: EvaluationInput /** Server-installed stable identity resolver; never accepted from an API payload. */ resolveToolInvocationId?: ( providerCallId: string | undefined, diff --git a/apps/sim/providers/typesafe/index.ts b/apps/sim/providers/typesafe/index.ts new file mode 100644 index 00000000000..044cb3f3abb --- /dev/null +++ b/apps/sim/providers/typesafe/index.ts @@ -0,0 +1,97 @@ +import type { StreamingExecution } from '@/executor/types' +import { getProviderDefaultModel, getProviderModels } from '@/providers/models' +import { createSettledAgentEventStream } from '@/providers/stream-events' +import { createStreamingExecution } from '@/providers/streaming-execution' +import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' +import { buildJevBody, parseJevResponse } from '@/providers/typesafe/schema' +import { requestJevEvaluation } from '@/providers/typesafe/transport' +import { calculateCost } from '@/providers/utils' + +export const typesafeProvider: ProviderConfig = { + id: 'typesafe', + name: 'TypeSafe', + description: 'Jev native evaluation models', + version: '1.0.0', + models: getProviderModels('typesafe'), + defaultModel: getProviderDefaultModel('typesafe'), + + async executeRequest(request: ProviderRequest): Promise { + if (!request.apiKey) throw new Error('API key is required for TypeSafe') + if (!request.evaluation) throw new Error('Jev requires evaluation state and questions') + if ( + request.messages?.length || + request.systemPrompt || + request.context || + request.tools?.length || + request.responseFormat || + request.previousInteractionId + ) { + throw new Error( + 'Jev accepts evaluation state and questions, not chat messages, tools, or response formats' + ) + } + + const body = buildJevBody( + { model: request.model, state: request.evaluation.state }, + request.evaluation.questions + ) + const start = Date.now() + const startTime = new Date(start).toISOString() + const result = parseJevResponse( + await requestJevEvaluation(body, request.apiKey, request.abortSignal), + body.questions + ) + const content = JSON.stringify(result.answers) + const tokens = { + input: result.usage.input_tokens, + output: result.usage.output_tokens, + total: result.usage.input_tokens + result.usage.output_tokens, + } + const cost = calculateCost(request.model, tokens.input, tokens.output) + if (request.stream) { + return createStreamingExecution({ + model: result.model, + providerStartTime: start, + providerStartTimeISO: startTime, + timing: { kind: 'simple', segmentName: result.model }, + initialTokens: tokens, + initialCost: cost, + isStreaming: true, + streamFormat: 'agent-events-v1', + createStream: ({ output, finalizeTiming }) => { + output.content = content + output.answers = result.answers + finalizeTiming() + return createSettledAgentEventStream(content) + }, + }) + } + const end = Date.now() + return { + content, + answers: result.answers, + model: result.model, + tokens, + cost, + timing: { + startTime, + endTime: new Date(end).toISOString(), + duration: end - start, + modelTime: end - start, + toolsTime: 0, + iterations: 1, + timeSegments: [ + { + type: 'model', + name: result.model, + startTime: start, + endTime: end, + duration: end - start, + tokens, + cost, + }, + ], + }, + } + }, +} diff --git a/apps/sim/providers/typesafe/schema.ts b/apps/sim/providers/typesafe/schema.ts new file mode 100644 index 00000000000..22f5fe5aae8 --- /dev/null +++ b/apps/sim/providers/typesafe/schema.ts @@ -0,0 +1,127 @@ +import { z } from 'zod' +import type { JevContent, JevEvaluationResult, JevQuestion } from '@/providers/typesafe/types' + +const contentSchema: z.ZodType = z.union([ + z.string(), + z.array(z.json()), + z.record(z.string(), z.json()), +]) + +const questionSchema: z.ZodType = z.discriminatedUnion('type', [ + z.object({ + type: z.literal('choice'), + instructions: contentSchema, + criteria: z + .record(z.string(), contentSchema.nullable()) + .refine( + (criteria) => Object.keys(criteria).length >= 1 && Object.keys(criteria).length <= 255, + 'Choice criteria must contain between 1 and 255 options' + ), + }), + z.object({ + type: z.literal('score'), + instructions: contentSchema, + criteria: z.array(contentSchema).min(2).max(10), + }), + z.object({ + type: z.literal('noul'), + instructions: contentSchema, + criteria: z + .object({ true: contentSchema.optional(), false: contentSchema.optional() }) + .strict() + .optional(), + }), +]) + +const questionsSchema = z + .record(z.string(), questionSchema) + .refine((questions) => Object.keys(questions).length > 0, 'Provide at least one question') + +const probabilitySchema = z.number().min(0).max(1) +const probabilitiesSchema = z.record(z.string(), probabilitySchema) +const responseSchema: z.ZodType = z.object({ + model: z.string(), + answers: z.record( + z.string(), + z.discriminatedUnion('type', [ + z.object({ + type: z.literal('choice'), + choice: z.string(), + probabilities: probabilitiesSchema, + confidence: probabilitySchema, + }), + z.object({ + type: z.literal('score'), + score: z.number(), + legend: z.record(z.string(), contentSchema), + probabilities: probabilitiesSchema, + confidence: probabilitySchema, + }), + z.object({ type: z.literal('noul'), noul: probabilitySchema }), + ]) + ), + usage: z.object({ + input_tokens: z.number().int().nonnegative(), + output_tokens: z.number().int().nonnegative(), + }), +}) + +export function parseJevJson(value: unknown, field: string): unknown { + if (typeof value !== 'string') return value + try { + return JSON.parse(value) + } catch { + throw new Error(`Jev ${field} must be valid JSON`) + } +} + +export function parseJevQuestions(questions: unknown) { + const parsedQuestions = questionsSchema.safeParse(parseJevJson(questions, 'questions')) + if (!parsedQuestions.success) { + throw new Error( + 'Invalid Jev questions: provide typed questions with instructions, 1–255 Choice options, 2–10 Score levels, or optional true/false Noul criteria' + ) + } + return parsedQuestions.data +} + +export function buildJevBody(params: { model: string; state: unknown }, questions: unknown) { + const parsedQuestions = parseJevQuestions(questions) + const state = contentSchema.safeParse(params.state) + if (!state.success) throw new Error('Jev state must be text, a JSON object, or an array') + return { + model: params.model, + state: state.data, + questions: parsedQuestions, + } +} + +export function parseJevResponse( + value: unknown, + questions: Record +): JevEvaluationResult { + const result = responseSchema.safeParse(value) + if (!result.success) throw new Error('TypeSafe returned an invalid Jev evaluation response') + const { answers } = result.data + if ( + Object.keys(answers).length !== Object.keys(questions).length || + Object.entries(questions).some( + ([id, question]) => !Object.hasOwn(answers, id) || answers[id].type !== question.type + ) + ) { + throw new Error( + 'TypeSafe returned Jev answers that do not match the requested question IDs and types' + ) + } + for (const [id, question] of Object.entries(questions)) { + const answer = answers[id] + if ( + question.type === 'choice' && + answer.type === 'choice' && + !Object.hasOwn(question.criteria, answer.choice) + ) { + throw new Error('TypeSafe returned a Jev Choice answer outside the requested options') + } + } + return result.data +} diff --git a/apps/sim/providers/typesafe/transport.ts b/apps/sim/providers/typesafe/transport.ts new file mode 100644 index 00000000000..137caf295ad --- /dev/null +++ b/apps/sim/providers/typesafe/transport.ts @@ -0,0 +1,67 @@ +import { interruptibleSleep } from '@sim/utils/helpers' +import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' +import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' +import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { PROVIDER_HEADERS_TIMEOUT_MS, PROVIDER_MAX_RETRIES } from '@/providers/transport' +import type { buildJevBody } from '@/providers/typesafe/schema' + +const MAX_EVALUATION_RESPONSE_BYTES = 10 * 1024 * 1024 +export const MAX_EVALUATION_REQUEST_BYTES = 10 * 1024 * 1024 + +class TypeSafeHttpError extends Error { + constructor( + readonly status: number, + readonly retryAfterMs: number | null + ) { + super(`TypeSafe evaluation failed (HTTP ${status})`) + this.name = 'TypeSafeHttpError' + } +} + +export async function requestJevEvaluation( + body: ReturnType, + apiKey: string, + abortSignal?: AbortSignal +): Promise { + const payload = stringifyBoundedJson(body, MAX_EVALUATION_REQUEST_BYTES) + if (payload === undefined) { + throw new Error('TypeSafe evaluation request exceeds the size or JSON complexity limit') + } + for (let attempt = 0; ; attempt++) { + abortSignal?.throwIfAborted() + const timeout = AbortSignal.timeout(PROVIDER_HEADERS_TIMEOUT_MS) + const signal = abortSignal ? AbortSignal.any([abortSignal, timeout]) : timeout + let response: Response | undefined + try { + signal.throwIfAborted() + response = await fetch('https://api.typesafe.ai/v1/systemone', { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + body: payload, + signal, + redirect: 'error', + }) + if (!response.ok) { + await consumeOrCancelBody(response) + throw new TypeSafeHttpError( + response.status, + parseRetryAfter(response.headers.get('retry-after')) + ) + } + return await readResponseJsonWithLimit(response, { + maxBytes: MAX_EVALUATION_RESPONSE_BYTES, + label: 'TypeSafe evaluation response', + signal, + }) + } catch (error) { + abortSignal?.throwIfAborted() + const retryable = + error instanceof TypeSafeHttpError + ? error.status === 408 || error.status === 429 || error.status >= 500 + : !response || timeout.aborted || error instanceof TypeError + if (!retryable || attempt >= PROVIDER_MAX_RETRIES) throw error + const retryAfterMs = error instanceof TypeSafeHttpError ? error.retryAfterMs : null + await interruptibleSleep(backoffWithJitter(attempt + 1, retryAfterMs), abortSignal) + } + } +} diff --git a/apps/sim/providers/typesafe/types.ts b/apps/sim/providers/typesafe/types.ts new file mode 100644 index 00000000000..6167f35fe47 --- /dev/null +++ b/apps/sim/providers/typesafe/types.ts @@ -0,0 +1,58 @@ +export type JevJsonValue = + | string + | number + | boolean + | null + | JevJsonValue[] + | { [key: string]: JevJsonValue } + +export type JevContent = string | JevJsonValue[] | { [key: string]: JevJsonValue } + +export type JevQuestion = + | { + type: 'choice' + instructions: JevContent + criteria: Record + } + | { type: 'score'; instructions: JevContent; criteria: JevContent[] } + | { + type: 'noul' + instructions: JevContent + criteria?: { true?: JevContent; false?: JevContent } + } + +export interface JevChoiceAnswer { + type: 'choice' + choice: string + probabilities: Record + confidence: number +} + +export interface JevScoreAnswer { + type: 'score' + score: number + legend: Record + probabilities: Record + confidence: number +} + +export interface JevNoulAnswer { + type: 'noul' + noul: number +} + +export type JevAnswer = JevChoiceAnswer | JevScoreAnswer | JevNoulAnswer + +export interface JevUsage { + input_tokens: number + output_tokens: number +} + +export interface JevResponseMetadata { + model: string + usage: JevUsage +} + +export interface JevEvaluationResult extends JevResponseMetadata { + answers: Record +} diff --git a/apps/sim/providers/typesafe/typesafe.test.ts b/apps/sim/providers/typesafe/typesafe.test.ts new file mode 100644 index 00000000000..e8f449125ff --- /dev/null +++ b/apps/sim/providers/typesafe/typesafe.test.ts @@ -0,0 +1,343 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + getHostedModels, + getModelCapabilities, + getProviderDefaultModel, + getProviderIcon, + getProviderModels, +} from '@/providers/models' +import { PROVIDER_MAX_RETRIES } from '@/providers/transport' +import type { ProviderRequest } from '@/providers/types' +import { typesafeProvider } from '@/providers/typesafe' +import { buildJevBody, parseJevResponse } from '@/providers/typesafe/schema' +import { MAX_EVALUATION_REQUEST_BYTES, requestJevEvaluation } from '@/providers/typesafe/transport' +import type { JevEvaluationResult, JevQuestion } from '@/providers/typesafe/types' +import { getProviderFromModel, shouldBillModelUsage } from '@/providers/utils' + +const QUESTIONS: Record = { + department: { + type: 'choice', + instructions: 'Which team?', + criteria: { billing: null, technical: 'Bugs' }, + }, + frustration: { + type: 'score', + instructions: 'How frustrated?', + criteria: ['Calm', 'Frustrated', 'Angry'], + }, + urgent: { type: 'noul', instructions: 'Is this urgent?' }, +} +const RESULT: JevEvaluationResult = { + model: 'jev-1.13.0', + answers: { + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: 0.88, technical: 0.12 }, + confidence: 0.81, + }, + frustration: { + type: 'score', + score: 1.05, + legend: { '0': 'Calm', '1': 'Frustrated', '2': 'Angry' }, + probabilities: { '0': 0, '1': 0.95, '2': 0.05 }, + confidence: 0.92, + }, + urgent: { type: 'noul', noul: 0.95 }, + }, + usage: { input_tokens: 318, output_tokens: 34 }, +} +const REQUEST: ProviderRequest = { + model: 'jev-1.13.0', + apiKey: 'test-key', + evaluation: { state: 'My payouts have been failing for three days.', questions: QUESTIONS }, +} +const fetchMock = vi.fn() + +describe('TypeSafe provider', () => { + beforeEach(() => { + fetchMock.mockReset().mockResolvedValue(Response.json(RESULT)) + vi.stubGlobal('fetch', fetchMock) + }) + afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() + vi.useRealTimers() + }) + + it.each(['jev-1.13.0', 'jev-latest', 'jev-preview'])( + 'routes hosted-capable %s through native evaluation', + async (model) => { + expect(getProviderFromModel(model)).toBe('typesafe') + expect(getHostedModels()).toContain(model) + expect(shouldBillModelUsage(model)).toBe(true) + expect(getModelCapabilities(model)).toMatchObject({ evaluation: true, memory: false }) + expect(getProviderIcon(model)).toBeDefined() + const result = await typesafeProvider.executeRequest({ ...REQUEST, model }) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.typesafe.ai/v1/systemone', + expect.objectContaining({ + method: 'POST', + redirect: 'error', + headers: { Authorization: 'Bearer test-key', 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, state: REQUEST.evaluation?.state, questions: QUESTIONS }), + }) + ) + expect(result).toMatchObject({ + content: JSON.stringify(RESULT.answers), + answers: RESULT.answers, + model: RESULT.model, + tokens: { input: 318, output: 34, total: 352 }, + timing: { iterations: 1, toolsTime: 0 }, + }) + } + ) + + it('defaults to the stable alias while retaining the pinned model', () => { + expect(getProviderDefaultModel('typesafe')).toBe('jev-latest') + expect(getProviderModels('typesafe')[0]).toBe('jev-latest') + expect(getProviderModels('typesafe')).toContain('jev-1.13.0') + }) + + it.each(['42', 'false', 'null', { text: 'Refund required' }, ['first', { second: true }]])( + 'preserves native state %j', + async (state) => { + await typesafeProvider.executeRequest({ + ...REQUEST, + evaluation: { state, questions: JSON.stringify(QUESTIONS) }, + }) + expect(JSON.parse(String(fetchMock.mock.calls[0][1]?.body)).state).toEqual(state) + } + ) + + it('delivers complete structured answers to streaming consumers', async () => { + const result = await typesafeProvider.executeRequest({ ...REQUEST, stream: true }) + if (!('execution' in result)) throw new Error('Expected streaming execution') + expect(result.execution.output).toMatchObject({ + answers: RESULT.answers, + content: JSON.stringify(RESULT.answers), + tokens: { total: 352 }, + }) + const reader = result.stream.getReader() + const events: unknown[] = [] + for (;;) { + const next = await reader.read() + if (next.done) break + events.push(next.value) + } + expect(JSON.stringify(events)).toContain('billing') + }) + + it.each([ + { apiKey: undefined }, + { evaluation: undefined }, + { messages: [{ role: 'user', content: 'Chat' }] }, + { responseFormat: { name: 'response', schema: {} } }, + ] satisfies Partial[])( + 'rejects incomplete or conversational requests before sending them', + async (override) => { + await expect(typesafeProvider.executeRequest({ ...REQUEST, ...override })).rejects.toThrow() + expect(fetchMock).not.toHaveBeenCalled() + } + ) + + it('does not echo upstream error bodies or credentials', async () => { + fetchMock.mockResolvedValue( + Response.json({ error: 'private provider context test-key' }, { status: 401 }) + ) + await expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow( + 'TypeSafe evaluation failed (HTTP 401)' + ) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it.each([408, 429, 500, 503])('retries HTTP %s and honors Retry-After', async (status) => { + vi.useFakeTimers() + fetchMock.mockResolvedValueOnce(new Response(null, { status, headers: { 'retry-after': '2' } })) + const result = typesafeProvider.executeRequest(REQUEST) + await vi.advanceTimersByTimeAsync(1999) + expect(fetchMock).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(await result).toMatchObject({ answers: RESULT.answers }) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it.each([400, 403, 422])('does not retry HTTP %s', async (status) => { + fetchMock.mockResolvedValueOnce(new Response(null, { status })) + await expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow(`HTTP ${status}`) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('retries connection failures within the shared provider retry budget', async () => { + vi.useFakeTimers() + fetchMock.mockRejectedValue(new TypeError('fetch failed')) + const result = expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow('fetch failed') + await vi.runAllTimersAsync() + await result + expect(fetchMock).toHaveBeenCalledTimes(PROVIDER_MAX_RETRIES + 1) + }) + + it('stops retrying repeated server failures', async () => { + vi.useFakeTimers() + fetchMock.mockImplementation(async () => new Response(null, { status: 503 })) + const result = expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow('HTTP 503') + await vi.runAllTimersAsync() + await result + expect(fetchMock).toHaveBeenCalledTimes(PROVIDER_MAX_RETRIES + 1) + }) + + it('gives a timed-out attempt a fresh deadline', async () => { + vi.useFakeTimers() + const deadline = new AbortController() + vi.spyOn(AbortSignal, 'timeout').mockReturnValueOnce(deadline.signal) + fetchMock.mockImplementationOnce(async () => { + deadline.abort(new DOMException('Timed out', 'TimeoutError')) + throw deadline.signal.reason + }) + const result = typesafeProvider.executeRequest(REQUEST) + await vi.runAllTimersAsync() + expect(await result).toMatchObject({ answers: RESULT.answers }) + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(fetchMock.mock.calls[1][1]?.signal?.aborted).toBe(false) + }) + + it('cancels immediately during Retry-After without sending another attempt', async () => { + vi.useFakeTimers() + const controller = new AbortController() + fetchMock.mockResolvedValueOnce( + new Response(null, { status: 429, headers: { 'retry-after': '30' } }) + ) + const result = expect( + typesafeProvider.executeRequest({ ...REQUEST, abortSignal: controller.signal }) + ).rejects.toThrow('Cancelled') + await vi.advanceTimersByTimeAsync(1) + controller.abort(new Error('Cancelled')) + await result + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + }) + + it('does not retry a malformed successful response', async () => { + fetchMock.mockResolvedValueOnce(new Response('{')) + await expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow() + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it.each([ + { label: 'ASCII values', character: 'x', bytes: 1, key: false }, + { label: 'UTF-8 values', character: '😀', bytes: 4, key: false }, + { label: 'control-character values', character: '\u0000', bytes: 6, key: false }, + { label: 'control-character keys', character: '\u0000', bytes: 6, key: true }, + { label: 'lone-surrogate values', character: '\ud800', bytes: 6, key: false }, + { label: 'lone-surrogate keys', character: '\ud800', bytes: 6, key: true }, + ])('rejects oversized $label before serialization or HTTP', async ({ character, bytes, key }) => { + const text = character.repeat(Math.ceil(MAX_EVALUATION_REQUEST_BYTES / bytes)) + const body = { + model: REQUEST.model, + state: key ? { [text]: null } : text, + questions: QUESTIONS, + } + const serialize = vi.spyOn(JSON, 'stringify') + await expect(requestJevEvaluation(body, 'test-key')).rejects.toThrow( + 'size or JSON complexity limit' + ) + expect(serialize).not.toHaveBeenCalled() + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('honors cancellation before network access', async () => { + await expect( + typesafeProvider.executeRequest({ ...REQUEST, abortSignal: AbortSignal.abort() }) + ).rejects.toThrow() + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('forwards cancellation to the request and bounds response allocation', async () => { + const controller = new AbortController() + fetchMock.mockImplementation(async (_url, init) => { + controller.abort() + expect(init?.signal?.aborted).toBe(true) + throw controller.signal.reason + }) + await expect( + typesafeProvider.executeRequest({ ...REQUEST, abortSignal: controller.signal }) + ).rejects.toThrow() + fetchMock.mockResolvedValue( + new Response('{}', { headers: { 'content-length': String(11 * 1024 * 1024) } }) + ) + await expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow('exceeds maximum size') + }) +}) + +describe('Jev native schema', () => { + it.each([ + {}, + { bad: { type: 'chat', instructions: 'Hello' } }, + { bad: { type: 'choice', instructions: 'Pick', criteria: {} } }, + { + bad: { + type: 'choice', + instructions: 'Pick', + criteria: Object.fromEntries(Array.from({ length: 256 }, (_, i) => [String(i), null])), + }, + }, + { bad: { type: 'score', instructions: 'Rate', criteria: ['One'] } }, + { bad: { type: 'score', instructions: 'Rate', criteria: Array(11).fill('Level') } }, + { bad: { type: 'noul', instructions: 'Test', criteria: { yes: 'Wrong key' } } }, + ])('rejects invalid question shape %j', (questions) => { + expect(() => buildJevBody({ model: REQUEST.model, state: 'Test' }, questions)).toThrow( + 'Invalid Jev questions' + ) + }) + + it.each([null, true, 42])('rejects invalid state %j', (state) => { + expect(() => buildJevBody({ model: REQUEST.model, state }, QUESTIONS)).toThrow('Jev state') + }) + + it('accepts structured instructions and all question types together', () => { + expect( + buildJevBody( + { model: REQUEST.model, state: { content: 'Test' } }, + { + ...QUESTIONS, + urgent: { + type: 'noul', + instructions: ['Is this urgent?'], + criteria: { true: { deadline: 'today' }, false: 'No deadline' }, + }, + } + ).questions.urgent.instructions + ).toEqual(['Is this urgent?']) + }) + + it('rejects malformed question JSON', () => { + expect(() => buildJevBody({ model: REQUEST.model, state: 'Test' }, '{')).toThrow('valid JSON') + }) + + it.each([ + {}, + { ...RESULT.answers, unexpected: { type: 'noul', noul: 0.1 } }, + { ...RESULT.answers, department: { type: 'noul', noul: 0.1 } }, + ])('rejects mismatched answer IDs or types', (answers) => { + expect(() => parseJevResponse({ ...RESULT, answers }, QUESTIONS)).toThrow('do not match') + }) + + it.each([ + { ...RESULT, usage: { input_tokens: -1, output_tokens: 0 } }, + { ...RESULT, answers: { urgent: { type: 'noul', noul: 1.1 } } }, + { model: 'jev-1.13.0', choices: [] }, + ])('rejects invalid provider responses', (value) => { + expect(() => parseJevResponse(value, QUESTIONS)).toThrow('invalid Jev evaluation response') + }) + + it.each(['unknown', 'toString'])('rejects an unrequested Choice option %s', (choice) => { + const answers = { + ...RESULT.answers, + department: { type: 'choice', choice, probabilities: { [choice]: 1 }, confidence: 1 }, + } + expect(() => parseJevResponse({ ...RESULT, answers }, QUESTIONS)).toThrow( + 'outside the requested options' + ) + }) +}) diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index c11f5831916..051655a11aa 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -160,6 +160,7 @@ export const providers: Record = { cerebras: buildProviderMetadata('cerebras'), groq: buildProviderMetadata('groq'), sakana: buildProviderMetadata('sakana'), + typesafe: buildProviderMetadata('typesafe'), nvidia: buildProviderMetadata('nvidia'), meta: buildProviderMetadata('meta'), zai: buildProviderMetadata('zai'), @@ -1175,10 +1176,17 @@ export function getApiKey(provider: string, model: string, userProvidedKey?: str const isZaiModel = provider === 'zai' const isXaiModel = provider === 'xai' const isKimiModel = provider === 'kimi' + const isTypeSafeModel = provider === 'typesafe' if ( isHosted && - (isOpenAIModel || isClaudeModel || isGeminiModel || isZaiModel || isXaiModel || isKimiModel) + (isOpenAIModel || + isClaudeModel || + isGeminiModel || + isZaiModel || + isXaiModel || + isKimiModel || + isTypeSafeModel) ) { const hostedModels = getHostedModels() const isModelHosted = hostedModels.some((m) => m.toLowerCase() === model.toLowerCase()) diff --git a/apps/sim/tools/types.ts b/apps/sim/tools/types.ts index ef4a63d92a5..29fee73c124 100644 --- a/apps/sim/tools/types.ts +++ b/apps/sim/tools/types.ts @@ -13,6 +13,7 @@ export type BYOKProviderId = | 'mistral' | 'zai' | 'kimi' + | 'typesafe' | 'xai' | 'fireworks' | 'together' diff --git a/packages/deployment-config/src/env-capabilities.ts b/packages/deployment-config/src/env-capabilities.ts index fc5ae477f58..9d8eb803a24 100644 --- a/packages/deployment-config/src/env-capabilities.ts +++ b/packages/deployment-config/src/env-capabilities.ts @@ -1545,6 +1545,7 @@ export const LLM_KEY_POOLS = { zai: { keys: ['ZAI_API_KEY_1', 'ZAI_API_KEY_2', 'ZAI_API_KEY_3'] }, xai: { keys: ['XAI_API_KEY_1', 'XAI_API_KEY_2', 'XAI_API_KEY_3'] }, kimi: { keys: ['KIMI_API_KEY_1', 'KIMI_API_KEY_2', 'KIMI_API_KEY_3'] }, + typesafe: { keys: ['TYPESAFE_API_KEY_1', 'TYPESAFE_API_KEY_2', 'TYPESAFE_API_KEY_3'] }, fireworks: { keys: ['FIREWORKS_API_KEY_1', 'FIREWORKS_API_KEY_2', 'FIREWORKS_API_KEY_3'], fallbackKey: 'FIREWORKS_API_KEY', diff --git a/packages/workflow-types/src/blocks.ts b/packages/workflow-types/src/blocks.ts index 10e08a588b0..4aa6311d5e9 100644 --- a/packages/workflow-types/src/blocks.ts +++ b/packages/workflow-types/src/blocks.ts @@ -59,6 +59,8 @@ export type SubBlockType = | 'modal' export interface OutputCondition { + /** Keep the output selectable when the compared value is resolved at execution time. */ + allowReference?: boolean field: string value: string | number | boolean | Array not?: boolean