From fc960e9b292189c5566ee9ae6dd21c0ec2b50242 Mon Sep 17 00:00:00 2001 From: kashish Date: Mon, 12 Oct 2026 17:39:06 +0530 Subject: [PATCH] fix: rename gen_ai.evaluate to gen_ai.decide across various tests and integrations --- .../suites/tracing/workers-ai/test.ts | 4 ++-- .../tests/send-to-sentry.test.ts | 12 ++++++------ .../node-eve/agent/tools/classify_ticket.ts | 6 +++--- .../test-applications/node-eve/tests/eve.test.ts | 6 +++--- .../node-mastra/tests/classifier.test.ts | 2 +- .../suites/tracing/langchain/v1/test.ts | 2 +- .../suites/tracing/mastra/test.ts | 2 +- .../suites/tracing/typesafe/test.ts | 4 ++-- .../tracing/vercelai/v6_v7/scenario-evaluate.mjs | 4 ++-- .../suites/tracing/vercelai/v6_v7/test.ts | 6 +++--- .../vercel-ai/vercel-ai-dc-subscriber.ts | 5 +---- .../test/ai/lib/tracing/workers-ai.test.ts | 2 +- .../test/integrations/vercel-ai/evaluate.test.ts | 5 ++--- 13 files changed, 28 insertions(+), 32 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts index 600062d6eb0a..17553064fa87 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts @@ -154,7 +154,7 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si is_segment: false, attributes: { [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, - [SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' }, + [SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' }, [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' }, @@ -200,7 +200,7 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal }) is_segment: false, attributes: { [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, - [SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' }, + [SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' }, [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' }, diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts index f5ce99ab2a0f..09b76f42e891 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts @@ -94,14 +94,14 @@ test('Sends a Workers AI gen_ai span to Sentry', async () => { }); }); -test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry', async () => { +test('Sends a Workers AI gen_ai.decide span for a TypeSafe Jev call to Sentry', async () => { const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-jev`, 200)); - console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`); + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); let spanId: string | undefined; await expect - .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS) + .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS) .toBeDefined(); await expect @@ -115,14 +115,14 @@ test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry' }); }); -test('Sends a Workers AI gen_ai.evaluate span for a Clef call to Sentry', async () => { +test('Sends a Workers AI gen_ai.decide span for a Clef call to Sentry', async () => { const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-clef`, 200)); - console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`); + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); let spanId: string | undefined; await expect - .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS) + .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS) .toBeDefined(); await expect diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/classify_ticket.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/classify_ticket.ts index 9dd65a11fd90..600d60cf6731 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/classify_ticket.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/classify_ticket.ts @@ -1,9 +1,9 @@ -import { experimental_evaluate } from 'ai'; +import { experimental_decide } from 'ai'; import { Experimental_EvaluationMockModelV4 } from 'ai/test'; import { defineTool } from 'eve/tools'; import { z } from 'zod'; -// Runs a TypeSafe Jev evaluation so the e2e test can assert its `gen_ai.evaluate` span. The model is +// Runs a TypeSafe Jev evaluation so the e2e test can assert its `gen_ai.decide` span. The model is // mocked because the e2e environment has no TypeSafe credentials. const jev = new Experimental_EvaluationMockModelV4({ provider: 'gateway', @@ -26,7 +26,7 @@ export default defineTool({ description: 'Classify a support ticket. Call this when asked to classify a ticket.', inputSchema: z.object({ ticket: z.string().min(1) }), async execute({ ticket }) { - const { answers } = await experimental_evaluate({ + const { answers } = await experimental_decide({ model: jev, state: ticket, questions: { diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts index 04ad1e5151f4..1d48e00220f9 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -135,9 +135,9 @@ test('captures errors thrown inside an eve tool', async ({ baseURL }) => { }); }); -test('captures a gen_ai.evaluate span for a Jev call inside an eve tool', async ({ baseURL }) => { +test('captures a gen_ai.decide span for a Jev call inside an eve tool', async ({ baseURL }) => { const traceSpansPromise = collectStreamedSpans(APP, spansOfTrace => - ['gen_ai.execute_tool', 'gen_ai.evaluate'].every(op => spansOfTrace.some(span => getSpanOp(span) === op)), + ['gen_ai.execute_tool', 'gen_ai.decide'].every(op => spansOfTrace.some(span => getSpanOp(span) === op)), ); const sessionId = await runAgentTurn( @@ -148,7 +148,7 @@ test('captures a gen_ai.evaluate span for a Jev call inside an eve tool', async const traceSpans = await traceSpansPromise; const executeTool = traceSpans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); - const evaluate = traceSpans.find(span => getSpanOp(span) === 'gen_ai.evaluate'); + const evaluate = traceSpans.find(span => getSpanOp(span) === 'gen_ai.decide'); expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('classify_ticket'); diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts index 6eee9bb68359..3dd562656dc6 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts @@ -19,7 +19,7 @@ const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify'); -test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => { +test('captures a Mastra classifier (Jev) evaluation as a gen_ai.decide span', async ({ baseURL }) => { const spansPromise = collectStreamedSpans( APP, spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan), diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index f14ccb93169d..925f2b09c738 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -241,7 +241,7 @@ describe('LangChain integration (v1)', () => { 'scenario-typesafe-classifier.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { - test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => { + test('records a TypeSafeClassifier run as a gen_ai.decide span', async () => { const runner = createRunner().ignore('event'); const spansPromise = runner.collectStreamedSpansUntilSegment('main'); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index d26ea2281428..328443ab94cb 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -221,7 +221,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { 'scenario-classifier.mjs', 'instrument.mjs', (createRunner, test) => { - test('maps a classifier evaluation to a gen_ai.evaluate span', async () => { + test('maps a classifier evaluation to a gen_ai.decide span', async () => { await createRunner() .expect({ span: container => { diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index e5feb8919745..65317253736f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -34,7 +34,7 @@ describe('TypeSafe integration', () => { .expect({ span: container => { const evaluateSpans = container.items.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', ); expect(evaluateSpans).toHaveLength(4); for (const span of evaluateSpans) { @@ -131,7 +131,7 @@ describe('TypeSafe integration', () => { .expect({ span: container => { const evaluateSpans = container.items.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', ); expect(evaluateSpans).toHaveLength(4); expect(evaluateSpans.filter(span => span.status === 'error')).toHaveLength(2); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs index 42fc5ead26c7..7bf2727c32d9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs @@ -1,10 +1,10 @@ import * as Sentry from '@sentry/node'; -import { experimental_evaluate } from 'ai'; +import { experimental_decide } from 'ai'; import { Experimental_EvaluationMockModelV4 } from 'ai/test'; async function run() { await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await experimental_evaluate({ + await experimental_decide({ model: new Experimental_EvaluationMockModelV4({ provider: 'gateway', modelId: 'typesafe-ai/jev', diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 893cea265ebb..eafd1cf3b782 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1275,7 +1275,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe ); }); -describe('Vercel AI integration experimental_evaluate', () => { +describe('Vercel AI integration experimental_decide', () => { afterAll(() => { cleanupChildProcesses(); }); @@ -1290,7 +1290,7 @@ describe('Vercel AI integration experimental_evaluate', () => { .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', )!; expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); expect(evaluateSpan.status).toBe('ok'); @@ -1361,7 +1361,7 @@ describe('Vercel AI integration experimental_evaluate', () => { .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', )!; expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index e9a75ade26b6..9f12ed7fbeb5 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -241,7 +241,6 @@ export type ChannelEventType = | 'embed' | 'embedMany' | 'rerank' - | 'experimental_evaluate' | 'experimental_decide'; /** @@ -484,8 +483,6 @@ export function createSpanFromMessage( } case 'rerank': return startGenAiSpan('rerank', modelId, baseAttributes); - // `ai` 7.0.128 renamed `experimental_evaluate` to `experimental_decide`; older 7.x still publishes the old name. - case 'experimental_evaluate': case 'experimental_decide': return startGenAiSpan('evaluate', modelId, { ...baseAttributes, @@ -688,7 +685,7 @@ function getOutputMessages( result: Record, finishReason: string | undefined, ): string | undefined { - if (type === 'experimental_evaluate' || type === 'experimental_decide') { + if (type === 'experimental_decide') { return stringify([{ type: 'evaluation', answers: withProviderConfidence(result) }]); } // `languageModelCall` exposes the response as a `content` parts array; top-level results expose diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts index f2784635c5f2..8c9cb34e5573 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts @@ -188,7 +188,7 @@ describe('instrumentWorkersAiClient', () => { expect(span.description).toBe('evaluate typesafe/jev'); expect(span.data).toEqual({ [SENTRY_ORIGIN]: 'auto.ai.cloudflare.workers_ai', - [SENTRY_OP]: 'gen_ai.evaluate', + [SENTRY_OP]: 'gen_ai.decide', [SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE]: 1, [SENTRY_SEGMENT_NAME_SOURCE]: 'custom', [GEN_AI_PROVIDER_NAME]: 'cloudflare.workers_ai', diff --git a/packages/server-utils/test/integrations/vercel-ai/evaluate.test.ts b/packages/server-utils/test/integrations/vercel-ai/evaluate.test.ts index 0101b19cf69b..678b4375a92e 100644 --- a/packages/server-utils/test/integrations/vercel-ai/evaluate.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/evaluate.test.ts @@ -27,8 +27,7 @@ describe('Vercel AI evaluate spans', () => { getMainCarrier().__SENTRY__ = undefined; }); - // `ai` 7.0.128 renamed the channel event from `experimental_evaluate` to `experimental_decide`. - it.each(['experimental_evaluate', 'experimental_decide'] as const)('creates an evaluate span for `%s`', type => { + it('creates a decide span for `experimental_decide`', () => { const message = { type, event: { @@ -49,7 +48,7 @@ describe('Vercel AI evaluate spans', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('evaluate typesafe-ai/jev'); - expect(json.op).toBe('gen_ai.evaluate'); + expect(json.op).toBe('gen_ai.decide'); expect(JSON.parse(json.data?.['gen_ai.output.messages'] as string)).toEqual([ { type: 'evaluation', answers: { wantsRefund: { type: 'boolean', probability: 0.99 } } }, ]);