Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .server-changes/orcarouter-ai-runs-provider-and-cost.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
area: webapp
type: feature
---

AI runs that go through OrcaRouter now show the correct downstream model provider and usage cost in the dashboard, just like OpenRouter runs.
4 changes: 2 additions & 2 deletions apps/webapp/app/components/metrics/ModelsFilter.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ interface ModelsFilterProps {
}

function modelIcon(system: string, model: string): ReactNode {
// For gateway/openrouter, derive provider from model prefix
// For gateway/openrouter/orcarouter, derive provider from model prefix
let provider = system.split(".")[0];
if (provider === "gateway" || provider === "openrouter") {
if (provider === "gateway" || provider === "openrouter" || provider === "orcarouter") {
if (model.includes("/")) {
provider = model.split("/")[0].replace(/-/g, "");
}
Expand Down
7 changes: 6 additions & 1 deletion apps/webapp/app/components/runs/v3/ai/aiHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export function formatDuration(ms: number): string {

/**
* Parse provider metadata from a JSON string.
* Handles Anthropic, Azure, OpenAI, Gateway, and OpenRouter formats.
* Handles Anthropic, Azure, OpenAI, Gateway, OpenRouter, and OrcaRouter formats.
*/
export function parseProviderMetadata(raw: unknown):
| {
Expand Down Expand Up @@ -74,6 +74,11 @@ export function parseProviderMetadata(raw: unknown):
resolvedProvider = str(rec(parsed.openrouter).provider);
}

// OrcaRouter: { orcarouter: { provider: "xAI" } } — same shape as OpenRouter
if (!resolvedProvider) {
resolvedProvider = str(rec(parsed.orcarouter).provider);
}

if (!serviceTier && !resolvedProvider && !gatewayCost && !responseId) return undefined;
return { serviceTier, resolvedProvider, gatewayCost, responseId };
} catch {
Expand Down
2 changes: 1 addition & 1 deletion apps/webapp/app/components/runs/v3/ai/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ export type AISpanData = {
// Categorical tags
finishReason?: string;
serviceTier?: string;
/** Resolved downstream provider for gateway/openrouter spans (e.g. "xAI", "mistral") */
/** Resolved downstream provider for gateway/openrouter/orcarouter spans (e.g. "xAI", "mistral") */
resolvedProvider?: string;
toolChoice?: string;
toolCount?: number;
Expand Down
13 changes: 12 additions & 1 deletion apps/webapp/app/routes/admin.llm-models.missing.$model.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ function getNestedObj(
// ---------------------------------------------------------------------------

type ProviderCostInfo = {
source: string; // "gateway" or "openrouter"
source: string; // "gateway", "openrouter", or "orcarouter"
cost: number;
inputTokens: number;
outputTokens: number;
Expand Down Expand Up @@ -350,6 +350,17 @@ function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[]
continue;
}
}

// OrcaRouter: { orcarouter: { usage: { cost: 4.94e-06 } } } — same shape as OpenRouter
const orc = getNestedObj(providerMeta, ["orcarouter"]);
const orcUsage = orc ? getNestedObj(orc, ["usage"]) : null;
if (orcUsage) {
const cost = Number(orcUsage.cost ?? 0);
if (cost > 0) {
costs.push({ source: "orcarouter", cost, inputTokens, outputTokens });
continue;
}
}
}

// Estimate per-token prices from aggregate costs if we have enough data
Expand Down
2 changes: 1 addition & 1 deletion apps/webapp/app/services/admin/missingLlmModels.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export async function getMissingLlmModels(
if (candidates.length === 0) return [];

// Filter out models that now have pricing in the database (added after spans were inserted).
// The registry's match() handles prefix stripping for gateway/openrouter models.
// The registry's match() handles prefix stripping for gateway/openrouter/orcarouter models.
if (!llmPricingRegistry || !llmPricingRegistry.isLoaded) return candidates;
const registry = llmPricingRegistry;
return candidates.filter((c) => !registry.match(c.model));
Expand Down
4 changes: 2 additions & 2 deletions apps/webapp/app/v3/querySchemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1039,7 +1039,7 @@ const llmMetricsSchema: TableSchema = {
cost_source: {
name: "cost_source",
...column("LowCardinality(String)", {
description: "Where cost data came from (registry, gateway, openrouter)",
description: "Where cost data came from (registry, gateway, openrouter, orcarouter)",
example: "registry",
}),
},
Expand Down Expand Up @@ -1133,7 +1133,7 @@ const llmMetricsSchema: TableSchema = {
provider_cost: {
name: "provider_cost",
...column("Decimal64(12)", {
description: "Provider-reported cost in USD (from gateway or openrouter)",
description: "Provider-reported cost in USD (from gateway, openrouter, or orcarouter)",
customRenderType: "costInDollars",
}),
},
Expand Down
21 changes: 21 additions & 0 deletions apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,27 @@ describe("enrichLlmMetrics — provider-reported cost", () => {
expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter");
});

it("prefers the orcarouter-reported cost over catalog pricing", () => {
setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 })));

const out = enrichOne(
makeEvent({
"gen_ai.system": "orcarouter",
"gen_ai.request.model": "orcarouter/fusion-flash",
"gen_ai.response.model": "openai/gpt-oss-120b",
"gen_ai.usage.input_tokens": 74,
"gen_ai.usage.output_tokens": 16,
"ai.response.providerMetadata": JSON.stringify({
orcarouter: { provider: "OpenAI", usage: { cost: 0.00000494 } },
}),
})
);

expect(out.properties["trigger.llm.total_cost"]).toBe(0.00000494);
expect(out.properties["trigger.llm.cost_source"]).toBe("orcarouter");
expect(out._llmMetrics?.costSource).toBe("orcarouter");
});

it("falls back to catalog pricing when no provider cost is reported", () => {
setLlmPricingRegistry(
registryReturning(
Expand Down
11 changes: 11 additions & 0 deletions apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,17 @@ function extractProviderCost(
}
}

// OrcaRouter: { orcarouter: { usage: { cost: 4.94e-06 } } } — same shape as OpenRouter
const orcarouter = meta.orcarouter;
if (orcarouter && typeof orcarouter === "object") {
const orc = orcarouter as Record<string, unknown>;
const usage = orc.usage;
if (usage && typeof usage === "object") {
const cost = Number((usage as Record<string, unknown>).cost ?? 0);
if (cost > 0) return { totalCost: cost, source: "orcarouter" };
}
}

return null;
}

Expand Down