Skip to content

fix(server-utils): update Vercel AI decide span naming #25275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kashish00208 wants to merge 1 commit into from
Closed

fix(server-utils): update Vercel AI decide span naming #25275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
is_segment: false,
attributes: {
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
[SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' },
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' },
Expand Down Expand Up @@ -200,7 +200,7 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal })
is_segment: false,
attributes: {
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
[SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' },
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' },
Expand Down
12 changes: 6 additions & 6 deletions ...2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,14 @@ test('Sends a Workers AI gen_ai span to Sentry', async () => {
});
});

test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry', async () => {
test('Sends a Workers AI gen_ai.decide span for a TypeSafe Jev call to Sentry', async () => {
const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-jev`, 200));

console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);

let spanId: string | undefined;
await expect
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS)
.toBeDefined();

await expect
Expand All @@ -115,14 +115,14 @@ test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry'
});
});

test('Sends a Workers AI gen_ai.evaluate span for a Clef call to Sentry', async () => {
test('Sends a Workers AI gen_ai.decide span for a Clef call to Sentry', async () => {
const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-clef`, 200));

console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);

let spanId: string | undefined;
await expect
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS)
.toBeDefined();

await expect
Expand Down
6 changes: 3 additions & 3 deletions dev-packages/e2e-tests/test-applications/node-eve/agent/tools/classify_ticket.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { experimental_evaluate } from 'ai';
import { experimental_decide } from 'ai';
import { Experimental_EvaluationMockModelV4 } from 'ai/test';
import { defineTool } from 'eve/tools';
import { z } from 'zod';

// Runs a TypeSafe Jev evaluation so the e2e test can assert its `gen_ai.evaluate` span. The model is
// Runs a TypeSafe Jev evaluation so the e2e test can assert its `gen_ai.decide` span. The model is
// mocked because the e2e environment has no TypeSafe credentials.
const jev = new Experimental_EvaluationMockModelV4({
provider: 'gateway',
Expand All @@ -26,7 +26,7 @@ export default defineTool({
description: 'Classify a support ticket. Call this when asked to classify a ticket.',
inputSchema: z.object({ ticket: z.string().min(1) }),
async execute({ ticket }) {
const { answers } = await experimental_evaluate({
const { answers } = await experimental_decide({
model: jev,
state: ticket,
questions: {
Expand Down
6 changes: 3 additions & 3 deletions dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,9 @@ test('captures errors thrown inside an eve tool', async ({ baseURL }) => {
});
});

test('captures a gen_ai.evaluate span for a Jev call inside an eve tool', async ({ baseURL }) => {
test('captures a gen_ai.decide span for a Jev call inside an eve tool', async ({ baseURL }) => {
const traceSpansPromise = collectStreamedSpans(APP, spansOfTrace =>
['gen_ai.execute_tool', 'gen_ai.evaluate'].every(op => spansOfTrace.some(span => getSpanOp(span) === op)),
['gen_ai.execute_tool', 'gen_ai.decide'].every(op => spansOfTrace.some(span => getSpanOp(span) === op)),
);

const sessionId = await runAgentTurn(
Expand All @@ -148,7 +148,7 @@ test('captures a gen_ai.evaluate span for a Jev call inside an eve tool', async
const traceSpans = await traceSpansPromise;

const executeTool = traceSpans.find(span => getSpanOp(span) === 'gen_ai.execute_tool');
const evaluate = traceSpans.find(span => getSpanOp(span) === 'gen_ai.evaluate');
const evaluate = traceSpans.find(span => getSpanOp(span) === 'gen_ai.decide');

expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('classify_ticket');

Expand Down
2 changes: 1 addition & 1 deletion dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span
const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean =>
getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify');

test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => {
test('captures a Mastra classifier (Jev) evaluation as a gen_ai.decide span', async ({ baseURL }) => {
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan),
Expand Down
2 changes: 1 addition & 1 deletion dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ describe('LangChain integration (v1)', () => {
'scenario-typesafe-classifier.mjs',
'instrument-with-pii.mjs',
(createRunner, test) => {
test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => {
test('records a TypeSafeClassifier run as a gen_ai.decide span', async () => {
const runner = createRunner().ignore('event');
const spansPromise = runner.collectStreamedSpansUntilSegment('main');

Expand Down
2 changes: 1 addition & 1 deletion dev-packages/node-integration-tests/suites/tracing/mastra/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => {
'scenario-classifier.mjs',
'instrument.mjs',
(createRunner, test) => {
test('maps a classifier evaluation to a gen_ai.evaluate span', async () => {
test('maps a classifier evaluation to a gen_ai.decide span', async () => {
await createRunner()
.expect({
span: container => {
Expand Down
4 changes: 2 additions & 2 deletions dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ describe('TypeSafe integration', () => {
.expect({
span: container => {
const evaluateSpans = container.items.filter(
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide',
);
expect(evaluateSpans).toHaveLength(4);
for (const span of evaluateSpans) {
Expand Down Expand Up @@ -131,7 +131,7 @@ describe('TypeSafe integration', () => {
.expect({
span: container => {
const evaluateSpans = container.items.filter(
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide',
);
expect(evaluateSpans).toHaveLength(4);
expect(evaluateSpans.filter(span => span.status === 'error')).toHaveLength(2);
Expand Down
4 changes: 2 additions & 2 deletions dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import * as Sentry from '@sentry/node';
import { experimental_evaluate } from 'ai';
import { experimental_decide } from 'ai';
import { Experimental_EvaluationMockModelV4 } from 'ai/test';

async function run() {
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
await experimental_evaluate({
await experimental_decide({
model: new Experimental_EvaluationMockModelV4({
provider: 'gateway',
modelId: 'typesafe-ai/jev',
Expand Down
6 changes: 3 additions & 3 deletions dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1275,7 +1275,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe
);
});

describe('Vercel AI integration experimental_evaluate', () => {
describe('Vercel AI integration experimental_decide', () => {
afterAll(() => {
cleanupChildProcesses();
});
Expand All @@ -1290,7 +1290,7 @@ describe('Vercel AI integration experimental_evaluate', () => {
.expect({
span: container => {
const evaluateSpan = container.items.find(
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide',
)!;
expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev');
expect(evaluateSpan.status).toBe('ok');
Expand Down Expand Up @@ -1361,7 +1361,7 @@ describe('Vercel AI integration experimental_evaluate', () => {
.expect({
span: container => {
const evaluateSpan = container.items.find(
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide',
)!;
expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
Expand Down
5 changes: 1 addition & 4 deletions packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,6 @@ export type ChannelEventType =
| 'embed'
| 'embedMany'
| 'rerank'
| 'experimental_evaluate'

@cursor cursor Bot Oct 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Old evaluate channel events ignored

High Severity

experimental_evaluate was dropped from the channel type, the start switch, and getOutputMessages. Older supported ai 7.x versions still publish that event name, so those decide/evaluate calls no longer create spans or record evaluation output messages.

Additional Locations (2)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit fc960e9. Configure here.

| 'experimental_decide';

/**
Expand Down Expand Up @@ -484,8 +483,6 @@ export function createSpanFromMessage(
}
case 'rerank':
return startGenAiSpan('rerank', modelId, baseAttributes);
// `ai` 7.0.128 renamed `experimental_evaluate` to `experimental_decide`; older 7.x still publishes the old name.
case 'experimental_evaluate':
case 'experimental_decide':
return startGenAiSpan('evaluate', modelId, {
...baseAttributes,
Expand Down Expand Up @@ -688,7 +685,7 @@ function getOutputMessages(
result: Record<string, unknown>,
finishReason: string | undefined,
): string | undefined {
if (type === 'experimental_evaluate' || type === 'experimental_decide') {
if (type === 'experimental_decide') {
return stringify([{ type: 'evaluation', answers: withProviderConfidence(result) }]);
}
// `languageModelCall` exposes the response as a `content` parts array; top-level results expose
Expand Down
2 changes: 1 addition & 1 deletion packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ describe('instrumentWorkersAiClient', () => {
expect(span.description).toBe('evaluate typesafe/jev');
expect(span.data).toEqual({
[SENTRY_ORIGIN]: 'auto.ai.cloudflare.workers_ai',
[SENTRY_OP]: 'gen_ai.evaluate',
[SENTRY_OP]: 'gen_ai.decide',

@cursor cursor Bot Oct 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unrelated tests expect decide op

Medium Severity

Workers AI, TypeSafe, and Cloudflare tests now assert sentry.op is gen_ai.decide, but those integrations still emit GEN_AI_EVALUATE (gen_ai.evaluate). This PR only changes the Vercel AI subscriber, so those suites fail without a matching production or conventions change.

Additional Locations (2)
Fix in Cursor Fix in Web

Triggered by project rule: PR Review Guidelines for Cursor Bot

Reviewed by Cursor Bugbot for commit fc960e9. Configure here.

[SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE]: 1,
[SENTRY_SEGMENT_NAME_SOURCE]: 'custom',
[GEN_AI_PROVIDER_NAME]: 'cloudflare.workers_ai',
Expand Down
5 changes: 2 additions & 3 deletions packages/server-utils/test/integrations/vercel-ai/evaluate.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ describe('Vercel AI evaluate spans', () => {
getMainCarrier().__SENTRY__ = undefined;
});

// `ai` 7.0.128 renamed the channel event from `experimental_evaluate` to `experimental_decide`.
it.each(['experimental_evaluate', 'experimental_decide'] as const)('creates an evaluate span for `%s`', type => {

@cursor cursor Bot Oct 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unit test leaves type unbound

Medium Severity

The it.each callback that supplied type was replaced with a plain it, but the message still uses the type shorthand. The test no longer sends experimental_decide and does not compile or run, so it does not cover the rename.

Fix in Cursor Fix in Web

Triggered by project rule: PR Review Guidelines for Cursor Bot

Reviewed by Cursor Bugbot for commit fc960e9. Configure here.

it('creates a decide span for `experimental_decide`', () => {
const message = {
type,
event: {
Expand All @@ -49,7 +48,7 @@ describe('Vercel AI evaluate spans', () => {

const json = spanToStaticSpanJSON(endedSpans[0]!);
expect(json.description).toBe('evaluate typesafe-ai/jev');
expect(json.op).toBe('gen_ai.evaluate');
expect(json.op).toBe('gen_ai.decide');
expect(JSON.parse(json.data?.['gen_ai.output.messages'] as string)).toEqual([
{ type: 'evaluation', answers: { wantsRefund: { type: 'boolean', probability: 0.99 } } },
]);
Expand Down
Loading