feat(jev): integrate native evaluation into Agent providers

Author: waleedlatif1

apps/docs/components/ui/icon-mapping.ts

@@ -286,7 +286,6 @@ import {
   TTSIcon,
   TwilioIcon,
   TypeformIcon,
-  TypeSafeIcon,
   UpstashIcon,
   UptimeRobotIcon,
   VantaIcon,
@@ -475,7 +474,6 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
   instantly: InstantlyIcon,
   intercom: IntercomIcon,
   intercom_v2: IntercomIcon,
-  jev: TypeSafeIcon,
   jina: JinaAIIcon,
   jira: JiraIcon,
   jira_service_management: JiraServiceManagementIcon,

apps/docs/content/docs/integrations/jev.mdx

@@ -131,7 +131,6 @@
     "infisical",
     "instantly",
     "intercom",
-    "jev",
     "jina",
     "jira",
     "jira_service_management",

apps/docs/content/docs/integrations/meta.json

@@ -31,6 +31,30 @@ For a custom cloud deployment, enter its provider prefix and model ID: `azure/my
 
 Ollama Cloud, OpenRouter, Fireworks, Together AI, Baseten, Ollama, vLLM, and LiteLLM load their available models from the configured provider. New models appear through that discovery without a Sim catalog release. You can also enter a namespaced ID directly, such as `ollama-cloud/deepseek-v4.1-flash`, `openrouter/provider/model`, or `ollama/my-local-model`. Provider prefixes are case-insensitive; the model ID after the prefix keeps its original casing.
 
+### Jev evaluation models
+
+Select `jev-1.13.0`, `jev-latest`, or `jev-preview` from TypeSafe in the Agent model selector, then enter your TypeSafe API key. These models use **State** and **Questions** in place of conversational messages. State accepts text or a reference to a JSON object or array. Questions is a JSON object keyed by the answer names you want:
+
+```json
+{
+  "route": {
+    "type": "choice",
+    "instructions": "Which team should handle this request?",
+    "criteria": { "billing": "Payments and invoices", "support": "Product issues" }
+  },
+  "urgency": {
+    "type": "score",
+    "instructions": "How urgent is the request?",
+    "criteria": ["Routine", "Soon", "Immediate"]
+  },
+  "resolved": { "type": "noul", "instructions": "Has the request been resolved?" }
+}
+```
+
+Read results from `<agent.answers>`. Each Choice answer includes `choice`, `probabilities`, and `confidence`; each Score answer includes `score`, `legend`, `probabilities`, and `confidence`; each Noul answer includes `noul`, a probability from 0 to 1. `content` contains the same answers as JSON text, and the standard model, token, timing, and cost outputs remain available. Use a Condition block to route on these results.
+
+Jev evaluates the supplied state in one request. Chat messages, files, tools, skills, conversation memory, response-format schemas, and chat model fallbacks are hidden for these models. Saved settings return when you switch back to a chat model. Bring your own key; Sim does not provide hosted Jev credits. TypeSafe documents a 64,000-token total request limit and a 32,000-token limit for state plus the longest question. See [TypeSafe's model documentation](https://docs.typesafe.ai/models) and [question formats](https://docs.typesafe.ai/api).
+
 ### Files
 
 Files for the model to read: images for a vision-capable model, or documents for text. Upload them on the block, or pass a file from an earlier block, such as an upload trigger or an [API](/workflows/blocks/api) response, with a connection tag.
@@ -104,7 +128,7 @@ Some settings live under advanced, or appear only for models that support them:
 - **Max output tokens.** Caps the response length. Defaults to the model's full limit.
 - **Reasoning effort / Thinking level.** For models with extended reasoning, how much the model thinks before answering. Higher is more thorough but slower and costs more tokens.
 - **Prompt caching.** For Anthropic Claude models, reuses the system prompt and tool definitions between runs instead of re-reading them every time. Cached input costs a tenth of the normal rate, but writing the cache costs 1.25x, so leave it off for one-off runs and turn it on when the same agent runs repeatedly. The cache covers a prefix only if it reaches 1,024 tokens (2,048 on Haiku) — below that Anthropic ignores it and nothing changes. Entries expire after five minutes of no use.
-- **API key.** Your key for the chosen provider. Hidden on hosted Sim, which supplies one.
+- **API key.** Your key for the chosen provider. Hidden when hosted Sim supplies a key for the selected model. Jev requires your own TypeSafe key.
 - **Fallback models.** An ordered list of up to five models to try when the request to the selected model fails, whether the provider is overloaded, rate-limited, or down. Sim tries the 2nd choice, then the 3rd, and so on, once each, and `<agent.model>` reports the model that answered. On hosted Sim, hosted models use your workspace's BYOK or platform credentials; local and self-hosted installations may still require a key. A model that needs its own key takes it from a workspace environment variable you pick on the row; a model on the same provider as the selected model reuses the block's key. A stored row key stops applying when its key field is hidden. Providers that require family-specific credentials, such as Vertex, can only be fallbacks for a selected model of the same family. The Auto model cannot be a fallback. A fallback runs with the selected model's settings where its provider accepts them: temperature and max output tokens are clamped to the fallback's limits, and when the fallback has a reasoning effort, thinking level, or verbosity setting that the selected model's value does not fit, the row shows that field so you can pick a value for it; leave it empty and the provider's default applies.
 - **Retry on fail.** Retries the selected model after a failure, up to a maximum number of tries with a wait between them. When its tries run out, the fallback models are tried in order, once each, with no wait before the first of them. A fallback is never retried. See [Retries and fallbacks](#retries-and-fallbacks) for how recorded tool results are reused and when a tool can execute again.
 

apps/docs/content/docs/workflows/blocks/agent.mdx

@@ -0,0 +1,72 @@
+import { describe, expect, it, vi } from 'vitest'
+import { evaluateSubBlockCondition } from '@/lib/workflows/subblocks/visibility'
+import { AgentBlock } from '@/blocks/blocks/agent'
+import { getAgentModelOptions, getModelOptions } from '@/blocks/utils'
+import { getBaseModelProviders } from '@/providers/models'
+import { Serializer } from '@/serializer'
+import { useProvidersStore } from '@/stores/providers/store'
+import type { BlockState } from '@/stores/workflows/workflow/types'
+
+vi.mock('@/blocks', async () => {
+  const { AgentBlock } = await import('@/blocks/blocks/agent')
+  return { getBlock: () => AgentBlock }
+})
+
+describe('Agent evaluation configuration', () => {
+  it.each(['jev-1.13.0', 'jev-latest', 'jev-preview'])(
+    'shows native fields and credentials for %s',
+    (model) => {
+      const visible = AgentBlock.subBlocks
+        .filter((field) => evaluateSubBlockCondition(field.condition, { model }))
+        .map((field) => field.id)
+      expect(visible).toEqual(['model', 'evaluationState', 'evaluationQuestions', 'apiKey'])
+    }
+  )
+
+  it('keeps evaluation inputs configurable for a model reference', () => {
+    for (const field of AgentBlock.subBlocks.filter((field) => field.id.startsWith('evaluation'))) {
+      expect(evaluateSubBlockCondition(field.condition, { model: '<start.model>' })).toBe(true)
+    }
+  })
+
+  it('shows Jev only in the model picker that supports evaluation inputs', () => {
+    useProvidersStore.getState().setProviderModels('base', Object.keys(getBaseModelProviders()))
+    expect(getAgentModelOptions().map((option) => option.id)).toContain('jev-1.13.0')
+    expect(getModelOptions().map((option) => option.id)).not.toContain('jev-1.13.0')
+  })
+
+  it.each([false, true])(
+    'serializes native fields without requiring messages, advanced=%s',
+    (advancedMode) => {
+      const values = {
+        model: 'jev-1.13.0',
+        apiKey: '{{TYPESAFE_API_KEY}}',
+        evaluationState: '42',
+        evaluationQuestions: '{"passed":{"type":"noul","instructions":"Did it pass?"}}',
+        messages: JSON.stringify([{ role: 'user', content: 'Old chat prompt' }]),
+      }
+      const block: BlockState = {
+        id: 'agent-test',
+        type: 'agent',
+        name: 'Evaluator',
+        position: { x: 0, y: 0 },
+        enabled: true,
+        advancedMode,
+        outputs: {},
+        subBlocks: Object.fromEntries(
+          Object.entries(values).map(([id, value]) => [
+            id,
+            { id, value, type: AgentBlock.subBlocks.find((field) => field.id === id)!.type },
+          ])
+        ),
+      }
+      const result = new Serializer().serializeWorkflow({ [block.id]: block }, [], {}, {}, true)
+      expect(result.blocks[0].config.tool).toBe('typesafe')
+      expect(result.blocks[0].config.params).toMatchObject({
+        evaluationState: '42',
+        evaluationQuestions: values.evaluationQuestions,
+      })
+      expect(result.blocks[0].config.params).not.toHaveProperty('messages')
+    }
+  )
+})