Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ test('instruments a Flue agent with no manual instrument() call', async () => {
await expect
.poll(() => fetchSpanAttributes(traceId!, chat!.event_id!), EVENT_POLLING_OPTIONS)
.toMatchObject({
origin: 'auto.ai.flue',
origin: 'auto.ai.pi_ai',
'gen_ai.provider.name': 'openrouter',
'gen_ai.usage.input_tokens': expect.anything(),
'gen_ai.cost.total_tokens': expect.anything(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ test('traces a PiHarness prompt as invoke_agent with chat, execute_tool and prov
// pi-ai sends the requests through `@anthropic-ai/sdk`, whose own integration must stay out so
// each request is reported once.
expect(spans.filter(span => String(span.attributes[SENTRY_ORIGIN]?.value).startsWith('auto.ai.'))).toEqual(
spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_durable'),
spans.filter(span =>
['auto.ai.pi_durable', 'auto.ai.pi_ai'].includes(String(span.attributes[SENTRY_ORIGIN]?.value)),
),
);

// One tool-calling response, then the answer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn'
expect(invokeAgent?.attributes?.['gen_ai.operation.name']?.value).toBe('invoke_agent');
expect(invokeAgent?.attributes?.['gen_ai.agent.name']?.value).toBe('Hello');

expect(chat?.attributes?.['sentry.origin']?.value).toBe('auto.ai.flue');
expect(chat?.attributes?.['sentry.origin']?.value).toBe('auto.ai.pi_ai');
expect(chat?.attributes?.['gen_ai.provider.name']?.value).toBe('openrouter');
expect(typeof chat?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe('number');
expect(typeof chat?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe('number');
// Flue computes cost itself; no provider SDK reports it.
// pi-ai computes cost itself; no provider SDK reports it.
expect(typeof chat?.attributes?.['gen_ai.cost.total_tokens']?.value).toBe('number');

expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('get_weather');
Expand Down Expand Up @@ -92,7 +92,7 @@ test('nests a manual span raised inside a tool under that tool span', async ({ b
expect(manualSpan?.parent_span_id).toBe(executeTool?.span_id);
});

// Flue's `model` operation is wrapped so the turn span is active for it, which is what puts the
// The pi-ai integration makes the chat span active for the request it sends, which is what puts the
// provider's HTTP call inside `chat` rather than beside it.
test('nests the provider HTTP call inside the chat span', async ({ baseURL }) => {
const spansPromise = collectStreamedSpans(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,16 @@ test('traces a run as invoke_agent with chat, execute_tool and provider spans',
// pi-ai sends the requests through `@anthropic-ai/sdk`, whose own integration must stay out so
// each request is reported once.
expect(spans.filter(span => String(span.attributes[SENTRY_ORIGIN]?.value).startsWith('auto.ai.'))).toEqual(
spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_durable'),
spans.filter(span =>
['auto.ai.pi_durable', 'auto.ai.pi_ai'].includes(String(span.attributes[SENTRY_ORIGIN]?.value)),
),
);

// One tool-calling response, then the answer.
expect(chats.length).toBeGreaterThanOrEqual(2);
for (const chat of chats) {
expect(chat.parent_span_id).toBe(agent.span_id);
expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable');
expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_ai');
expect(chat.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter');
expect(chat.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('anthropic/claude-haiku-4.5');
// pi-durable retries a provider error inside the run; such a request has no usage and no HTTP span
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@ describe.runIf(supports({ min: 22 }))('Flue integration', () => {
expect(root.is_segment).toBe(true);

// Counted rather than looked up: the interceptor skips the submission wrapper
// operation, so one dispatch opens exactly one agent span, and each turn and tool call
// is spanned once. `find` passes just as happily on a duplicate.
expect(spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.flue')).toHaveLength(4);
// operation, so one dispatch opens exactly one agent span, and each tool call is spanned
// once. Each turn is spanned once by the pi-ai integration: a provider SDK reporting the
// same request would add a span. `find` passes just as happily on a duplicate.
expect(spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.flue')).toHaveLength(2);
expect(spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_ai')).toHaveLength(2);

const agents = spans.filter(span => span.name === 'invoke_agent Hello');
const chats = spans.filter(span => span.name === 'chat faux-model');
Expand All @@ -62,17 +64,17 @@ describe.runIf(supports({ min: 22 }))('Flue integration', () => {
expect(conversationId).toEqual(expect.any(String));

// Both turns, not just the first: they leave the provider by different paths (a tool
// call, then a final answer) and resolve their parent through separate tracker lookups.
// call, then a final answer).
for (const chat of chats) {
expect(chat.attributes[SENTRY_OP]?.value).toBe(GEN_AI_CHAT);
expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.flue');
expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_ai');
expect(chat.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('chat');
expect(chat.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId);
expect(chat.parent_span_id).toBe(agent.span_id);
expect(chat.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0);
expect(chat.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBeGreaterThan(0);
expect(chat.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBeGreaterThan(0);
// Flue computes cost itself; no provider SDK reports it. The faux provider prices
// pi-ai computes cost itself; no provider SDK reports it. The faux provider prices
// every model at zero, so this only proves the attribute is mapped.
expect(chat.attributes[GEN_AI_COST_TOTAL_TOKENS]?.value).toEqual(expect.any(Number));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,4 @@ Sentry.init({
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [Sentry.piAiIntegration()],
});
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,12 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
const agent = spans.find(span => span.is_segment)!;
expect(agent.name).toBe('invoke_agent');
expect(spans.filter(span => span.name === 'chat faux-model')).toHaveLength(2);
// One span per run, request and tool call: a tool wrapped twice would add a span.
// One span per run and tool call, and one per request from the pi-ai integration: a tool
// wrapped twice, or a request reported by a provider SDK too, would add a span.
expect(spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_durable')).toHaveLength(
5,
3,
);
expect(spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_ai')).toHaveLength(2);

expect(agent.attributes[SENTRY_OP]?.value).toBe(GEN_AI_INVOKE_AGENT);
expect(agent.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable');
Expand All @@ -77,7 +79,7 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
for (const chat of chats) {
expect(chat.parent_span_id).toBe(agent.span_id);
expect(chat.attributes[SENTRY_OP]?.value).toBe(GEN_AI_CHAT);
expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable');
expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_ai');
expect(chat.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('faux');
expect(chat.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('faux-model');
expect(chat.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId);
Expand Down Expand Up @@ -491,10 +493,8 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
})
.expect({
span: container => {
// 2. The provider threw: the generation faulted.
const chat = container.items.find(span => span.name === 'chat faux-model')!;
expect(chat.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toBeUndefined();
expect(chat.attributes[SENTRY_STATUS_MESSAGE]?.value).toBe('internal_error');
// 2. The provider threw in front of pi-ai, so there is no request to report.
expect(container.items.find(span => span.name === 'chat faux-model')).toBeUndefined();
const agent = container.items.find(span => span.is_segment)!;
expect(agent.attributes[SENTRY_STATUS_MESSAGE]?.value).toBe('internal_error');
},
Expand Down
12 changes: 5 additions & 7 deletions packages/server-utils/src/ai/flue/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,9 @@ export const FLUE_INSTRUMENTATION_KEY = Symbol.for('sentry.flue.instrumentation'
/**
* The Flue execution operations we act on.
*
* Only `AGENT` is spanned from the interceptor, and only at the outermost depth: Flue drives one
* LLM call through many `MODEL` operations (one per stream read), and its `AGENT` operation nests
* inside itself once per submission. The turn span is driven from the observation stream instead,
* where `turn_start`/`turn` are exactly one-to-one with a model call. `MODEL` and `TOOL` are
* intercepted only to make the already-open span active for the duration of the operation.
* Only `AGENT` is spanned from the interceptor, and only at the outermost depth: its operation nests
* inside itself once per submission. `TOOL` is intercepted to make the tool span `observe` opened
* active, and `MODEL` to hand the conversation to the pi-ai integration, which spans the model call.
*/
export const FLUE_OPERATION = {
AGENT: 'agent',
Expand All @@ -34,8 +32,8 @@ export const FLUE_OPERATION = {
} as const;

/**
* Cap on tracked turn and tool spans, matching `MAX_TRACKED_MASTRA_SPANS`. Both maps are keyed off
* an id that is only removed when the matching end observation arrives; a stream that is abandoned
* Cap on tracked tool spans and admissions, matching `MAX_TRACKED_MASTRA_SPANS`. Both maps are keyed
* off an id that is only removed when the matching end observation arrives; a run that is abandoned
* mid-turn never emits one, so without a cap the map grows for the lifetime of the process.
*/
export const MAX_TRACKED_FLUE_SPANS = 1000;
Loading
Loading