Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 50 additions & 11 deletions apps/api/src/routes/agent-business-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const state = vi.hoisted(() => ({
contexts: [] as Record<string, unknown>[],
accessible: vi.fn(),
errors: vi.fn(),
events: vi.fn(),
sessionUserId: "user-synthetic" as string | null,
sessionOrg: "org-synthetic" as string | null,
apiKeyId: null as string | null,
Expand Down Expand Up @@ -224,7 +225,9 @@ vi.mock("@databuddy/ai/agents/cache", () => ({
vi.mock("@databuddy/ai/lib/ai-logger", () => ({
getAILogger: () => ({ wrap: (model: MockLanguageModelV3) => model }),
}));
vi.mock("@databuddy/ai/lib/databuddy", () => ({ trackAgentEvent: () => {} }));
vi.mock("@databuddy/ai/lib/databuddy", () => ({
trackAgentEvent: state.events,
}));
vi.mock("@databuddy/ai/lib/tracing", () => ({
captureError: state.errors,
mergeWideEvent: () => {},
Expand Down Expand Up @@ -308,6 +311,7 @@ beforeEach(() => {
state.contexts.length = 0;
state.read.mockReset();
state.errors.mockReset();
state.events.mockReset();
state.accessible.mockReset();
state.read.mockImplementation(async () => ({
profile: state.profile,
Expand Down Expand Up @@ -858,6 +862,41 @@ const mixedHeaders = {
"x-api-key": "dbdy_inert_selected_key",
};

describe("failure telemetry identity", () => {
it.each([
{ route: "ask", identity: "key", key: selectedKey },
{ route: "chat", identity: "key", key: selectedKey },
{ route: "ask", identity: "session", key: null },
{ route: "chat", identity: "session", key: null },
])("attributes failure telemetry to the selected $identity on /$route", async ({
route,
key,
}) => {
state.apiKey = key;
const organizationId = key?.organizationId ?? "org-synthetic";
const userId = key ? `apikey:${key.id}` : "user-synthetic";
state.billing.mockRejectedValueOnce(
new BillingUnavailableError("Synthetic billing outage")
);
const response = await (route === "ask"
? ask({ organizationId })
: chat({ organizationId }));
expect(response.status).toBe(503);
expect(state.events).toHaveBeenCalledWith(
"agent_activity",
expect.objectContaining({
action: "chat_error",
organization_id: organizationId,
user_id: userId,
})
);
expect(state.errors).toHaveBeenCalledWith(
expect.any(Error),
expect.objectContaining({ agent_user_id: userId })
);
});
});

describe("selected agent identity through native HTTP", () => {
it.each([
{ stream: false, keyOwner: "key-owner-synthetic" },
Expand All @@ -875,17 +914,13 @@ describe("selected agent identity through native HTTP", () => {
expect(response.status).toBe(200);
expect(await response.text()).toContain("Synthetic answer.");
const options = (stream ? state.stream : state.ask).mock.calls[0]?.[0];
expect(options.actor).toMatchObject({
type: "api_key",
apiKey: state.apiKey,
userId: null,
});
expect(options.principal).toMatchObject({
apiKey: state.apiKey,
organizationId: selectedKey.organizationId,
userId: keyOwner,
});
expect(options.actor.requestHeaders.get("cookie")).toBeNull();
expect(options.actor.requestHeaders.get("x-api-key")).toBe(
expect(options.principal.requestHeaders.get("cookie")).toBeNull();
expect(options.principal.requestHeaders.get("x-api-key")).toBe(
mixedHeaders["x-api-key"]
);
expect(options.abortSignal).toBeInstanceOf(AbortSignal);
Expand Down Expand Up @@ -1020,9 +1055,13 @@ describe("selected agent identity through native HTTP", () => {
const response = await askResponse({}, { cookie: mixedHeaders.cookie });
expect(response.status).toBe(200);
expect(await response.text()).toContain("Synthetic answer.");
const actor = state.ask.mock.calls[0]?.[0].actor;
expect(actor).toMatchObject({ type: "session", userId: "user-synthetic" });
expect(actor.requestHeaders.get("cookie")).toBe(mixedHeaders.cookie);
const principal = state.ask.mock.calls[0]?.[0].principal;
expect(principal).toMatchObject({
apiKey: null,
organizationId: "org-synthetic",
userId: "user-synthetic",
});
expect(principal.requestHeaders.get("cookie")).toBe(mixedHeaders.cookie);
});

it.each<{
Expand Down
81 changes: 38 additions & 43 deletions apps/api/src/routes/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ import { AGENT_THINKING_LEVELS, AGENT_TIERS } from "@databuddy/ai/agents/types";
import { type AgentModelKey, models } from "@databuddy/ai/config/models";
import {
AgentError,
type AgentRequestInput,
askDatabuddyAgent,
type DatabuddyAgentActor,
type DatabuddyAgentOptions,
prepareAgentRequest,
streamDatabuddyAgent,
toAgentErrorResponse,
Expand Down Expand Up @@ -106,36 +106,42 @@ function agentActor(

function agentFailure(
error: unknown,
event: {
chatId: string;
organizationId: string | null;
source: "api" | "dashboard";
userId: string | null;
websiteId?: string | null;
}
source: "api" | "dashboard",
chatId: string,
{
activeOrganizationId,
apiKey,
body,
user,
}: AgentAuth & { body: { organizationId?: string; websiteId?: string } }
): Response {
if (error instanceof AgentError && error.status < 500) {
mergeWideEvent({ agent_rejected: error.code });
return toAgentErrorResponse(error);
}
const errorType = getErrorName(error);
const userId = apiKey ? `apikey:${apiKey.id}` : user?.id;
trackAgentEvent("agent_activity", {
action: "chat_error",
source: event.source,
source,
agent_type: AGENT_TYPE,
error_type: errorType,
organization_id: event.organizationId,
user_id: event.userId,
website_id: event.websiteId ?? null,
organization_id:
apiKey?.organizationId ??
body.organizationId ??
activeOrganizationId ??
null,
user_id: userId ?? null,
website_id: body.websiteId ?? null,
});
captureError(error, {
agent_error: true,
agent_type: AGENT_TYPE,
agent_chat_id: event.chatId,
...(event.websiteId ? { agent_website_id: event.websiteId } : {}),
...(event.userId ? { agent_user_id: event.userId } : {}),
agent_chat_id: chatId,
...(body.websiteId ? { agent_website_id: body.websiteId } : {}),
...(userId ? { agent_user_id: userId } : {}),
error_type: errorType,
source: event.source,
source,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
});
return toAgentErrorResponse(error);
}
Expand Down Expand Up @@ -499,28 +505,26 @@ export const agent = new Elysia({ prefix: "/v1/agent" })
mergeWideEvent({ agent_chat_id: conversationId, source: "api" });

try {
const agentRequest: AgentRequestInput = {
const principal = await prepareAgentRequest({
actor: agentActor(
{ activeOrganizationId, apiKey, user },
agentHeaders
),
organizationId: body.organizationId,
rateLimit: "agent:ask",
};
const principal = await prepareAgentRequest(agentRequest);
});
mergeWideEvent({
organization_id: principal.organizationId,
agent_user_id: user?.id ?? `apikey:${apiKey?.id}`,
});
const options = {
...agentRequest,
const options: DatabuddyAgentOptions = {
abortSignal: request.signal,
conversationId,
input: body.question,
mutationMode: "dry-run" as const,
output: "markdown" as const,
mutationMode: "dry-run",
output: "markdown",
principal,
source: "api" as const,
source: "api",
timezone: body.timezone,
};
if (body.stream) {
Expand All @@ -530,15 +534,11 @@ export const agent = new Elysia({ prefix: "/v1/agent" })
}
return await askDatabuddyAgent(options);
} catch (error) {
return agentFailure(error, {
chatId: conversationId,
organizationId:
body.organizationId ??
activeOrganizationId ??
apiKey?.organizationId ??
null,
source: "api",
userId: user?.id ?? null,
return agentFailure(error, "api", conversationId, {
activeOrganizationId,
apiKey,
body,
user,
});
}
},
Expand Down Expand Up @@ -1074,16 +1074,11 @@ export const agent = new Elysia({ prefix: "/v1/agent" })
}
return response;
} catch (error) {
return agentFailure(error, {
chatId,
organizationId:
body.organizationId ??
activeOrganizationId ??
apiKey?.organizationId ??
null,
source: "dashboard",
userId: user?.id ?? null,
websiteId: body.websiteId,
return agentFailure(error, "dashboard", chatId, {
activeOrganizationId,
apiKey,
body,
user,
});
}
})();
Expand Down
11 changes: 4 additions & 7 deletions apps/api/src/routes/mcp.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { isApiKeyPresent } from "@databuddy/api-keys/resolve";
import { resolveAgentOrganizationId } from "@databuddy/ai/agent";
import {
createMcpErrorResponse,
createMcpUnauthorizedResponse,
Expand Down Expand Up @@ -249,12 +248,10 @@ export const mcp = new Elysia({ name: "mcp" })
user,
apiKey,
oauthAccessToken: null,
organizationId: resolveAgentOrganizationId({
activeOrganizationId: user
? session?.session.activeOrganizationId
: null,
apiKey,
}),
organizationId:
apiKey?.organizationId ??
(user ? session?.session.activeOrganizationId : null) ??
null,
};
})
.onBeforeHandle(({ user, apiKey, oauthAccessToken, set }) => {
Expand Down
2 changes: 1 addition & 1 deletion apps/slack/src/slack/blocks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
componentsToBlocks,
componentToBlocks,
} from "@/slack/blocks";
import { buildAnalyticsInstructionsForMcp } from "../../../../packages/ai/src/ai/prompts/analytics";
import { buildAnalyticsInstructionsForMcp } from "@databuddy/ai/prompts/analytics";

function firstBlock(spec: ComponentSpec): Block {
const block = componentToBlocks(spec)[0];
Expand Down
11 changes: 5 additions & 6 deletions packages/ai/src/agent/conversation-history.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ mock.module("./slack-relevance", () => ({

function answer(options: RunMcpAgentOptions): string {
runs.push(options);
return `Answer: ${options.question}`;
return `Answer: ${options.input}`;
}
mock.module("../ai/mcp/run-agent", () => ({
runMcpAgent: async (options: RunMcpAgentOptions) => answer(options),
Expand Down Expand Up @@ -73,7 +73,6 @@ function principalFor(apiKey: ApiKeyRow | null): AgentPrincipal {
};
}
const options: DatabuddyAgentOptions = {
actor: { type: "api_key", apiKey: key },
principal: principalFor(key),
conversationId: "slack-T_TEST-C_TEST-111_000",
input: "First question",
Expand Down Expand Up @@ -108,8 +107,8 @@ test.each([
});
await invoke({ ...options, input: "Third question" });

expect(runs.map((run) => run.priorMessages?.length ?? 0)).toEqual([0, 2, 4]);
expect(runs[1]?.priorMessages).toEqual([
expect(runs.map((run) => run.history?.length ?? 0)).toEqual([0, 2, 4]);
expect(runs[1]?.history).toEqual([
{ role: "user", content: "First question" },
{ role: "assistant", content: "Answer: First question" },
]);
Expand All @@ -130,7 +129,7 @@ test.each([
] as const)("isolates Slack history by %s", async (_scope, isolated) => {
await askDatabuddyAgent(options);
await askDatabuddyAgent({ ...options, ...isolated });
expect(runs.at(-1)?.priorMessages).toBeUndefined();
expect(runs.at(-1)?.history).toBeUndefined();
});

test.each([
Expand All @@ -145,7 +144,7 @@ test.each([
await askDatabuddyAgent(input);
await askDatabuddyAgent({ ...input, memoryUserId: "slack-T_TEST-U_B" });
await askDatabuddyAgent(input);
expect(runs.map((run) => run.priorMessages?.length ?? 0)).toEqual([0, 0, 2]);
expect(runs.map((run) => run.history?.length ?? 0)).toEqual([0, 0, 2]);
});

test("renders component JSON as markdown for markdown output", async () => {
Expand Down
2 changes: 1 addition & 1 deletion packages/ai/src/agent/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ const AGENT_ERRORS = {
},
} as const;

export type AgentErrorCode = keyof typeof AGENT_ERRORS;
type AgentErrorCode = keyof typeof AGENT_ERRORS;

export class AgentError extends Error {
readonly code: AgentErrorCode;
Expand Down
Loading
Loading