From 3d5535a9833c2308b89dd64f85dd6067cdf6f2d4 Mon Sep 17 00:00:00 2001 From: armyluki-wq Date: Thu, 27 Aug 2026 03:16:42 +0000 Subject: [PATCH] feat: recognize OrcaRouter as a named AI provider in dashboard analytics Add first-class recognition for OrcaRouter alongside the existing gateway/OpenRouter handling: resolve the downstream provider from `orcarouter.provider` span metadata, extract provider-reported usage cost, and derive the model-filter icon from the model prefix. This mirrors the OpenRouter integration so OrcaRouter runs show the same provider/cost fidelity in the dashboard instead of being treated as anonymous custom endpoints. --- .../orcarouter-ai-runs-provider-and-cost.md | 6 ++++++ .../app/components/metrics/ModelsFilter.tsx | 4 ++-- .../app/components/runs/v3/ai/aiHelpers.ts | 7 ++++++- .../webapp/app/components/runs/v3/ai/types.ts | 2 +- .../admin.llm-models.missing.$model.tsx | 13 +++++++++++- .../services/admin/missingLlmModels.server.ts | 2 +- apps/webapp/app/v3/querySchemas.ts | 4 ++-- .../enrichCreatableEvents.server.test.ts | 21 +++++++++++++++++++ .../v3/utils/enrichCreatableEvents.server.ts | 11 ++++++++++ 9 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 .server-changes/orcarouter-ai-runs-provider-and-cost.md diff --git a/.server-changes/orcarouter-ai-runs-provider-and-cost.md b/.server-changes/orcarouter-ai-runs-provider-and-cost.md new file mode 100644 index 00000000000..d1f41986d1b --- /dev/null +++ b/.server-changes/orcarouter-ai-runs-provider-and-cost.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +AI runs that go through OrcaRouter now show the correct downstream model provider and usage cost in the dashboard, just like OpenRouter runs. diff --git a/apps/webapp/app/components/metrics/ModelsFilter.tsx b/apps/webapp/app/components/metrics/ModelsFilter.tsx index 7803c5a8fdf..bf8ea489468 100644 --- a/apps/webapp/app/components/metrics/ModelsFilter.tsx +++ b/apps/webapp/app/components/metrics/ModelsFilter.tsx @@ -28,9 +28,9 @@ interface ModelsFilterProps { } function modelIcon(system: string, model: string): ReactNode { - // For gateway/openrouter, derive provider from model prefix + // For gateway/openrouter/orcarouter, derive provider from model prefix let provider = system.split(".")[0]; - if (provider === "gateway" || provider === "openrouter") { + if (provider === "gateway" || provider === "openrouter" || provider === "orcarouter") { if (model.includes("/")) { provider = model.split("/")[0].replace(/-/g, ""); } diff --git a/apps/webapp/app/components/runs/v3/ai/aiHelpers.ts b/apps/webapp/app/components/runs/v3/ai/aiHelpers.ts index 2de9ccb68cc..4bfe88a6379 100644 --- a/apps/webapp/app/components/runs/v3/ai/aiHelpers.ts +++ b/apps/webapp/app/components/runs/v3/ai/aiHelpers.ts @@ -30,7 +30,7 @@ export function formatDuration(ms: number): string { /** * Parse provider metadata from a JSON string. - * Handles Anthropic, Azure, OpenAI, Gateway, and OpenRouter formats. + * Handles Anthropic, Azure, OpenAI, Gateway, OpenRouter, and OrcaRouter formats. */ export function parseProviderMetadata(raw: unknown): | { @@ -74,6 +74,11 @@ export function parseProviderMetadata(raw: unknown): resolvedProvider = str(rec(parsed.openrouter).provider); } + // OrcaRouter: { orcarouter: { provider: "xAI" } } — same shape as OpenRouter + if (!resolvedProvider) { + resolvedProvider = str(rec(parsed.orcarouter).provider); + } + if (!serviceTier && !resolvedProvider && !gatewayCost && !responseId) return undefined; return { serviceTier, resolvedProvider, gatewayCost, responseId }; } catch { diff --git a/apps/webapp/app/components/runs/v3/ai/types.ts b/apps/webapp/app/components/runs/v3/ai/types.ts index adb085714af..9abd2284738 100644 --- a/apps/webapp/app/components/runs/v3/ai/types.ts +++ b/apps/webapp/app/components/runs/v3/ai/types.ts @@ -72,7 +72,7 @@ export type AISpanData = { // Categorical tags finishReason?: string; serviceTier?: string; - /** Resolved downstream provider for gateway/openrouter spans (e.g. "xAI", "mistral") */ + /** Resolved downstream provider for gateway/openrouter/orcarouter spans (e.g. "xAI", "mistral") */ resolvedProvider?: string; toolChoice?: string; toolCount?: number; diff --git a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx index fdecc168c7d..9e09cb723b1 100644 --- a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx +++ b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx @@ -292,7 +292,7 @@ function getNestedObj( // --------------------------------------------------------------------------- type ProviderCostInfo = { - source: string; // "gateway" or "openrouter" + source: string; // "gateway", "openrouter", or "orcarouter" cost: number; inputTokens: number; outputTokens: number; @@ -350,6 +350,17 @@ function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[] continue; } } + + // OrcaRouter: { orcarouter: { usage: { cost: 4.94e-06 } } } — same shape as OpenRouter + const orc = getNestedObj(providerMeta, ["orcarouter"]); + const orcUsage = orc ? getNestedObj(orc, ["usage"]) : null; + if (orcUsage) { + const cost = Number(orcUsage.cost ?? 0); + if (cost > 0) { + costs.push({ source: "orcarouter", cost, inputTokens, outputTokens }); + continue; + } + } } // Estimate per-token prices from aggregate costs if we have enough data diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts index 2ad5bffb520..e4b1ea79737 100644 --- a/apps/webapp/app/services/admin/missingLlmModels.server.ts +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -73,7 +73,7 @@ export async function getMissingLlmModels( if (candidates.length === 0) return []; // Filter out models that now have pricing in the database (added after spans were inserted). - // The registry's match() handles prefix stripping for gateway/openrouter models. + // The registry's match() handles prefix stripping for gateway/openrouter/orcarouter models. if (!llmPricingRegistry || !llmPricingRegistry.isLoaded) return candidates; const registry = llmPricingRegistry; return candidates.filter((c) => !registry.match(c.model)); diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 690bbaf5396..085a270a7e6 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -1039,7 +1039,7 @@ const llmMetricsSchema: TableSchema = { cost_source: { name: "cost_source", ...column("LowCardinality(String)", { - description: "Where cost data came from (registry, gateway, openrouter)", + description: "Where cost data came from (registry, gateway, openrouter, orcarouter)", example: "registry", }), }, @@ -1133,7 +1133,7 @@ const llmMetricsSchema: TableSchema = { provider_cost: { name: "provider_cost", ...column("Decimal64(12)", { - description: "Provider-reported cost in USD (from gateway or openrouter)", + description: "Provider-reported cost in USD (from gateway, openrouter, or orcarouter)", customRenderType: "costInDollars", }), }, diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts index 9deaa51307a..dcbff121c9e 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts @@ -156,6 +156,27 @@ describe("enrichLlmMetrics — provider-reported cost", () => { expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter"); }); + it("prefers the orcarouter-reported cost over catalog pricing", () => { + setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 }))); + + const out = enrichOne( + makeEvent({ + "gen_ai.system": "orcarouter", + "gen_ai.request.model": "orcarouter/fusion-flash", + "gen_ai.response.model": "openai/gpt-oss-120b", + "gen_ai.usage.input_tokens": 74, + "gen_ai.usage.output_tokens": 16, + "ai.response.providerMetadata": JSON.stringify({ + orcarouter: { provider: "OpenAI", usage: { cost: 0.00000494 } }, + }), + }) + ); + + expect(out.properties["trigger.llm.total_cost"]).toBe(0.00000494); + expect(out.properties["trigger.llm.cost_source"]).toBe("orcarouter"); + expect(out._llmMetrics?.costSource).toBe("orcarouter"); + }); + it("falls back to catalog pricing when no provider cost is reported", () => { setLlmPricingRegistry( registryReturning( diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 73bf8bf31b2..1a10f725f5f 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -372,6 +372,17 @@ function extractProviderCost( } } + // OrcaRouter: { orcarouter: { usage: { cost: 4.94e-06 } } } — same shape as OpenRouter + const orcarouter = meta.orcarouter; + if (orcarouter && typeof orcarouter === "object") { + const orc = orcarouter as Record; + const usage = orc.usage; + if (usage && typeof usage === "object") { + const cost = Number((usage as Record).cost ?? 0); + if (cost > 0) return { totalCost: cost, source: "orcarouter" }; + } + } + return null; }