From 49f4d032a5ead09ef05a8425670a02a9f5fd98b7 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 12 Aug 2026 00:17:06 +0200 Subject: [PATCH 1/2] refactor: harden Effect lint boundaries --- .oxlintrc.effect.json | 14 +- apps/api/scripts/bench-queries.ts | 60 +- apps/api/src/mcp/lib/query-warehouse.ts | 6 +- apps/api/src/mcp/tools/diagnose-service.ts | 4 +- apps/api/src/mcp/tools/error-detail.ts | 4 +- apps/api/src/mcp/tools/explore-attributes.ts | 8 +- apps/api/src/mcp/tools/find-errors.ts | 4 +- .../get-instrumentation-recommendations.ts | 4 +- .../mcp/tools/get-service-top-operations.ts | 4 +- apps/api/src/mcp/tools/list-services.ts | 4 +- apps/api/src/mcp/tools/mine-log-patterns.ts | 4 +- apps/api/src/mcp/tools/search-logs.ts | 4 +- apps/api/src/mcp/tools/service-map.ts | 4 +- apps/api/src/routes/v1/billing.http.test.ts | 90 ++- apps/api/src/routes/v1/billing.http.ts | 8 +- apps/api/src/routes/v1/observability.http.ts | 18 +- .../services/alerts/AlertDeliveryDispatch.ts | 660 +++++++++--------- apps/api/src/services/auth/auth.test.ts | 13 +- .../api/src/services/billing/autumn-client.ts | 16 +- .../services/errors/error-tick-persistence.ts | 4 +- .../CloudflareAnalyticsService.ts | 7 +- .../integrations/CloudflareApiImpl.ts | 8 +- .../PlanetScaleConnectionService.ts | 45 +- .../scrape-check-retention.test.ts | 6 +- .../integrations/scrape-check-retention.ts | 6 +- .../org/OrgClickHouseSettingsService.test.ts | 8 +- .../org/OrgClickHouseSettingsService.ts | 24 +- .../warehouse/WarehouseQueryService.ts | 11 +- apps/api/src/worker.ts | 2 + .../ClickHouseSchemaApplyWorkflow.run.ts | 17 +- .../InvestigationFanoutWorkflow.run.ts | 9 +- apps/api/src/workflows/agent-pass.test.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 2 + apps/cli/src/server/otlp/encode.ts | 2 + apps/electric-sync/src/worker.ts | 2 + .../src/api/warehouse/custom-charts.test.ts | 2 +- apps/web/src/api/warehouse/services.test.ts | 2 +- .../api/warehouse/timeseries-adapters.test.ts | 2 +- .../alerts-overview-model.registry.test.ts | 7 +- examples/alchemy-maple/src/Api.ts | 1 + examples/effect-todo/server/TodoService.ts | 8 +- packages/alchemy-maple/test/providers.test.ts | 14 +- packages/clickhouse-cli/src/client.ts | 2 + .../db/src/schema/planetscale-connections.ts | 4 +- packages/db/src/schema/scrape-targets.ts | 5 +- .../src/caching/bucket-cache.hit-rate.test.ts | 18 +- .../src/caching/bucket-cache.test.ts | 32 +- 47 files changed, 617 insertions(+), 564 deletions(-) diff --git a/.oxlintrc.effect.json b/.oxlintrc.effect.json index 6faf6a9e0..125e0b3ce 100644 --- a/.oxlintrc.effect.json +++ b/.oxlintrc.effect.json @@ -32,7 +32,7 @@ "effecttsgo/effect-map-flatten": "warn", "effecttsgo/effect-map-void": "warn", "effecttsgo/effect-succeed-with-void": "warn", - "effecttsgo/extends-native-error": "warn", + "effecttsgo/extends-native-error": "error", "effecttsgo/flat-map-to-map": "warn", "effecttsgo/floating-effect": "error", "effecttsgo/floating-effect-in-vitest": "error", @@ -42,11 +42,11 @@ "effecttsgo/global-date": "warn", "effecttsgo/global-date-in-effect": "warn", "effecttsgo/global-error-in-effect-catch": "warn", - "effecttsgo/global-error-in-effect-failure": "warn", + "effecttsgo/global-error-in-effect-failure": "error", "effecttsgo/global-fetch": "warn", - "effecttsgo/global-fetch-in-effect": "warn", + "effecttsgo/global-fetch-in-effect": "error", "effecttsgo/global-random": "warn", - "effecttsgo/global-random-in-effect": "warn", + "effecttsgo/global-random-in-effect": "error", "effecttsgo/global-timers": "warn", "effecttsgo/global-timers-in-effect": "warn", "effecttsgo/instance-of-schema": "warn", @@ -63,7 +63,7 @@ "effecttsgo/missing-return-yield-star": "error", "effecttsgo/missing-star-in-yield-effect-gen": "error", "effecttsgo/multiple-catch-tag": "error", - "effecttsgo/multiple-effect-provide": "warn", + "effecttsgo/multiple-effect-provide": "error", "effecttsgo/nested-effect-gen-yield": "warn", "effecttsgo/new-promise": "warn", "effecttsgo/new-schema-class": "off", @@ -92,11 +92,11 @@ "effecttsgo/scope-in-layer-effect": "warn", "effecttsgo/service-not-as-class": "warn", "effecttsgo/strict-boolean-expressions": "off", - "effecttsgo/strict-effect-provide": "warn", + "effecttsgo/strict-effect-provide": "error", "effecttsgo/sync-to-succeed": "error", "effecttsgo/try-catch-in-effect-gen": "warn", "effecttsgo/unknown-in-effect-catch": "warn", - "effecttsgo/unnecessary-effect-gen": "warn", + "effecttsgo/unnecessary-effect-gen": "error", "effecttsgo/unnecessary-fail-yieldable-error": "warn", "effecttsgo/unnecessary-arrow-block": "off", "effecttsgo/unnecessary-pipe": "warn", diff --git a/apps/api/scripts/bench-queries.ts b/apps/api/scripts/bench-queries.ts index 7d21be1ea..a5ab80617 100644 --- a/apps/api/scripts/bench-queries.ts +++ b/apps/api/scripts/bench-queries.ts @@ -35,6 +35,7 @@ import { Schema, } from "effect" import { Argument, Command, Flag } from "effect/unstable/cli" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" import { BunRuntime, BunServices } from "@effect/platform-bun" import { CH } from "@maple/query-engine" import * as Integrations from "@maple/query-engine-integrations" @@ -235,6 +236,7 @@ interface ClickHouseShape { export class ClickHouse extends Context.Service()("bench/ClickHouse", { make: Effect.gen(function* () { const { clickhouse } = yield* BenchConfig + const httpClient = yield* HttpClient.HttpClient const requireConfig = Option.match(clickhouse, { onNone: () => @@ -265,23 +267,22 @@ export class ClickHouse extends Context.Service()(" url.searchParams.set("wait_end_of_query", "1") const start = performance.now() - const response = yield* Effect.tryPromise({ - try: (signal) => - fetch(url, { - method: "POST", - headers: { - Authorization: authHeader(cfg), - "Content-Type": "text/plain; charset=utf-8", - }, - body: sql, - signal, - }), - catch: (cause) => new HttpRequestError({ url: cfg.url, message: String(cause) }), - }) - const body = yield* Effect.tryPromise({ - try: () => response.text(), - catch: (cause) => new HttpRequestError({ url: cfg.url, message: String(cause) }), - }) + const request = HttpClientRequest.post(url, { + headers: { + Authorization: authHeader(cfg), + "Content-Type": "text/plain; charset=utf-8", + }, + }).pipe(HttpClientRequest.bodyText(sql)) + const response = yield* httpClient + .execute(request) + .pipe( + Effect.mapError( + (cause) => new HttpRequestError({ url: cfg.url, message: String(cause) }), + ), + ) + const body = yield* response.text.pipe( + Effect.mapError((cause) => new HttpRequestError({ url: cfg.url, message: String(cause) })), + ) const wallMs = performance.now() - start return { @@ -347,6 +348,7 @@ interface TinybirdShape { export class Tinybird extends Context.Service()("bench/Tinybird", { make: Effect.gen(function* () { const { tinybird } = yield* BenchConfig + const httpClient = yield* HttpClient.HttpClient const requireConfig = Option.match(tinybird, { onNone: () => @@ -364,14 +366,19 @@ export class Tinybird extends Context.Service()("bench/ const query = Effect.fn("Tinybird.query")(function* (sql: string) { const cfg = yield* requireConfig const url = `${cfg.host}/v0/sql?q=${encodeURIComponent(sql)}` - const response = yield* Effect.tryPromise({ - try: (signal) => fetch(url, { headers: { Authorization: `Bearer ${cfg.token}` }, signal }), - catch: (cause) => new HttpRequestError({ url: cfg.host, message: String(cause) }), - }) - const text = yield* Effect.tryPromise({ - try: () => response.text(), - catch: (cause) => new HttpRequestError({ url: cfg.host, message: String(cause) }), + const request = HttpClientRequest.get(url, { + headers: { Authorization: `Bearer ${cfg.token}` }, }) + const response = yield* httpClient + .execute(request) + .pipe( + Effect.mapError( + (cause) => new HttpRequestError({ url: cfg.host, message: String(cause) }), + ), + ) + const text = yield* response.text.pipe( + Effect.mapError((cause) => new HttpRequestError({ url: cfg.host, message: String(cause) })), + ) if (!response.ok) { return yield* Effect.fail( new UpstreamStatusError({ @@ -956,7 +963,10 @@ const rootCommand = Command.make("bench-queries").pipe( Command.withSubcommands([fetchCommand, runCommand, inspectCommand, compareCommand]), ) -const BenchLive = Layer.mergeAll(ClickHouse.layer, Tinybird.layer, BunServices.layer) +const BenchServicesLive = Layer.mergeAll(ClickHouse.layer, Tinybird.layer).pipe( + Layer.provide(FetchHttpClient.layer), +) +const BenchLive = Layer.mergeAll(BenchServicesLive, BunServices.layer) Command.run(rootCommand, { version: "0.1.0" }).pipe( // Application root: this is the one runtime boundary that owns the complete layer graph. diff --git a/apps/api/src/mcp/lib/query-warehouse.ts b/apps/api/src/mcp/lib/query-warehouse.ts index 69ecacc49..57b1696e5 100644 --- a/apps/api/src/mcp/lib/query-warehouse.ts +++ b/apps/api/src/mcp/lib/query-warehouse.ts @@ -7,7 +7,7 @@ import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error" import { McpAuthMissingError } from "@/mcp/tools/types" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { WarehouseExecutor } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export class CurrentMcpTenant extends Context.Service()( "@maple/api/mcp/CurrentMcpTenant", @@ -35,12 +35,12 @@ export const resolveHttpMcpTenant = Effect.gen(function* () { return yield* resolveMcpTenantContext(nativeReq) }) -/** Infrastructure binding: resolves tenant and provides WarehouseExecutor layer. */ +/** Infrastructure binding: resolves the tenant and installs its WarehouseExecutor facade. */ export const withTenantExecutor = Effect.fn("withTenantExecutor")(function* ( effect: Effect.Effect, ) { const tenant = yield* CurrentMcpTenant - return yield* Effect.provide(effect, makeWarehouseExecutorFromTenant(tenant)) + return yield* effect.pipe(provideWarehouseExecutorFromTenant(tenant)) }) export const queryWarehouse = Effect.fn("queryWarehouse")(function* ( diff --git a/apps/api/src/mcp/tools/diagnose-service.ts b/apps/api/src/mcp/tools/diagnose-service.ts index e5d827ebf..9c4a00f88 100644 --- a/apps/api/src/mcp/tools/diagnose-service.ts +++ b/apps/api/src/mcp/tools/diagnose-service.ts @@ -7,7 +7,7 @@ import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error" import { Array as Arr, Effect, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { diagnoseService } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerDiagnoseServiceTool(server: McpToolRegistrar) { server.tool( @@ -28,7 +28,7 @@ export function registerDiagnoseServiceTool(server: McpToolRegistrar) { timeRange: { startTime: st, endTime: et }, environment: environment ?? undefined, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("service_overview")), ) diff --git a/apps/api/src/mcp/tools/error-detail.ts b/apps/api/src/mcp/tools/error-detail.ts index 4e44fbf24..2d8a8f55e 100644 --- a/apps/api/src/mcp/tools/error-detail.ts +++ b/apps/api/src/mcp/tools/error-detail.ts @@ -13,7 +13,7 @@ import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error" import { Array as Arr, Effect, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { errorDetail } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerErrorDetailTool(server: McpToolRegistrar) { server.tool( @@ -49,7 +49,7 @@ export function registerErrorDetailTool(server: McpToolRegistrar) { includeTimeseries: include_timeseries ?? false, limit: limit ?? 5, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("error_detail_traces")), ) diff --git a/apps/api/src/mcp/tools/explore-attributes.ts b/apps/api/src/mcp/tools/explore-attributes.ts index 08cb52868..93e8e683a 100644 --- a/apps/api/src/mcp/tools/explore-attributes.ts +++ b/apps/api/src/mcp/tools/explore-attributes.ts @@ -9,7 +9,7 @@ import { Array as Arr, Effect, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { formatNextSteps } from "@/mcp/lib/next-steps" import { exploreAttributeKeys, exploreAttributeValues } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerExploreAttributesTool(server: McpToolRegistrar) { server.tool( @@ -43,7 +43,7 @@ export function registerExploreAttributesTool(server: McpToolRegistrar) { const lim = clampLimit(params.limit, { defaultValue: 50, max: 500 }) const scope = (params.scope ?? "span") as "span" | "resource" const tenant = yield* CurrentMcpTenant - const executorLayer = makeWarehouseExecutorFromTenant(tenant) + const provideExecutor = provideWarehouseExecutorFromTenant(tenant) const mapError = toMcpQueryError("explore_attributes") const baseInput = { @@ -56,7 +56,7 @@ export function registerExploreAttributesTool(server: McpToolRegistrar) { if (params.key) { const values = yield* exploreAttributeValues({ ...baseInput, key: params.key }).pipe( - Effect.provide(executorLayer), + provideExecutor, Effect.mapError(mapError), ) @@ -174,7 +174,7 @@ export function registerExploreAttributesTool(server: McpToolRegistrar) { // List keys for traces or metrics const keys = yield* exploreAttributeKeys(baseInput).pipe( - Effect.provide(executorLayer), + provideExecutor, Effect.mapError(mapError), ) diff --git a/apps/api/src/mcp/tools/find-errors.ts b/apps/api/src/mcp/tools/find-errors.ts index a7c9e3f70..53351f1b0 100644 --- a/apps/api/src/mcp/tools/find-errors.ts +++ b/apps/api/src/mcp/tools/find-errors.ts @@ -7,7 +7,7 @@ import { formatNextSteps } from "@/mcp/lib/next-steps" import { Array as Arr, Effect, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { findErrors } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerFindErrorsTool(server: McpToolRegistrar) { server.tool( @@ -30,7 +30,7 @@ export function registerFindErrorsTool(server: McpToolRegistrar) { environment: environment ?? undefined, limit: limit ?? 20, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("errors_by_type")), ) diff --git a/apps/api/src/mcp/tools/get-instrumentation-recommendations.ts b/apps/api/src/mcp/tools/get-instrumentation-recommendations.ts index 5748ecba7..a60f0cb76 100644 --- a/apps/api/src/mcp/tools/get-instrumentation-recommendations.ts +++ b/apps/api/src/mcp/tools/get-instrumentation-recommendations.ts @@ -15,7 +15,7 @@ import { resolveTimeRange } from "@/mcp/lib/time" import { RecommendationIssueService } from "@/services/errors/RecommendationIssueService" import { RecommendationIssueStatus, type RecommendationIssueKind } from "@maple/domain/http" import { exploreAttributeKeys } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" const decodeStatus = Schema.decodeUnknownOption(RecommendationIssueStatus) @@ -155,7 +155,7 @@ export function registerGetInstrumentationRecommendationsTool(server: McpToolReg timeRange: { startTime: range.st, endTime: range.et }, limit: 500, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("get_instrumentation_recommendations")), Effect.option, ) diff --git a/apps/api/src/mcp/tools/get-service-top-operations.ts b/apps/api/src/mcp/tools/get-service-top-operations.ts index 00e767fd9..3db09f899 100644 --- a/apps/api/src/mcp/tools/get-service-top-operations.ts +++ b/apps/api/src/mcp/tools/get-service-top-operations.ts @@ -16,7 +16,7 @@ import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error" import { Effect, Option, Schema } from "effect" import { topOperations } from "@maple/query-engine/observability" import { TracesMetric } from "@maple/query-engine" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" const decodeTracesMetric = Schema.decodeUnknownOption(TracesMetric) @@ -66,7 +66,7 @@ export function registerGetServiceTopOperationsTool(server: McpToolRegistrar) { timeRange: { startTime: st, endTime: et }, limit: resolvedLimit, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("top_operations")), ) diff --git a/apps/api/src/mcp/tools/list-services.ts b/apps/api/src/mcp/tools/list-services.ts index 680a39c04..3c57f7d10 100644 --- a/apps/api/src/mcp/tools/list-services.ts +++ b/apps/api/src/mcp/tools/list-services.ts @@ -7,7 +7,7 @@ import { createDualContent } from "@/mcp/lib/structured-output" import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error" import { Array as Arr, Effect, Schema } from "effect" import { listServices } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerListServicesTool(server: McpToolRegistrar) { server.tool( @@ -30,7 +30,7 @@ export function registerListServicesTool(server: McpToolRegistrar) { timeRange: { startTime: st, endTime: et }, environment: environment ?? undefined, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("service_overview")), ) diff --git a/apps/api/src/mcp/tools/mine-log-patterns.ts b/apps/api/src/mcp/tools/mine-log-patterns.ts index 26d7a4c2a..dee7c5340 100644 --- a/apps/api/src/mcp/tools/mine-log-patterns.ts +++ b/apps/api/src/mcp/tools/mine-log-patterns.ts @@ -7,7 +7,7 @@ import { formatNextSteps } from "@/mcp/lib/next-steps" import { Effect, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { mineLogPatterns } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerMineLogPatternsTool(server: McpToolRegistrar) { server.tool( @@ -60,7 +60,7 @@ export function registerMineLogPatternsTool(server: McpToolRegistrar) { sampleSize, limit: lim, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("mine_log_patterns")), ) diff --git a/apps/api/src/mcp/tools/search-logs.ts b/apps/api/src/mcp/tools/search-logs.ts index 4cf306fef..1e33d6b42 100644 --- a/apps/api/src/mcp/tools/search-logs.ts +++ b/apps/api/src/mcp/tools/search-logs.ts @@ -8,7 +8,7 @@ import { formatNextSteps } from "@/mcp/lib/next-steps" import { Effect, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { searchLogs } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerSearchLogsTool(server: McpToolRegistrar) { server.tool( @@ -64,7 +64,7 @@ export function registerSearchLogsTool(server: McpToolRegistrar) { limit: lim, offset: off, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError(toMcpQueryError("search_logs")), ) diff --git a/apps/api/src/mcp/tools/service-map.ts b/apps/api/src/mcp/tools/service-map.ts index f350012e2..ff583cdee 100644 --- a/apps/api/src/mcp/tools/service-map.ts +++ b/apps/api/src/mcp/tools/service-map.ts @@ -6,7 +6,7 @@ import { formatNextSteps } from "@/mcp/lib/next-steps" import { Array as Arr, Effect, HashSet, Order, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { serviceMap } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" export function registerServiceMapTool(server: McpToolRegistrar) { server.tool( @@ -32,7 +32,7 @@ export function registerServiceMapTool(server: McpToolRegistrar) { service: service_name ?? undefined, environment: environment ?? undefined, }).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.mapError( (e) => new McpQueryError({ message: e.message, pipeName: "service_dependencies", cause: e }), diff --git a/apps/api/src/routes/v1/billing.http.test.ts b/apps/api/src/routes/v1/billing.http.test.ts index 72d3416a9..cc850d62b 100644 --- a/apps/api/src/routes/v1/billing.http.test.ts +++ b/apps/api/src/routes/v1/billing.http.test.ts @@ -1,5 +1,6 @@ -import { assert, describe, it } from "@effect/vitest" +import { assert, describe, it, layer } from "@effect/vitest" import { Effect, Schema } from "effect" +import { FetchHttpClient, HttpClient } from "effect/unstable/http" import { type EdgeCacheBackend, makeEdgeCacheService, makeMemoryBackend } from "@maple/cache" import { CUSTOMER_CACHE_BUCKET, @@ -44,46 +45,45 @@ const activePlanResponse = { } const noPlanResponse = { id: ORG, subscriptions: [] } -describe("updateCustomerBillingControls", () => { - it("uses Autumn's canonical customer update route and v2.3 wire shape", async () => { - const originalFetch = globalThis.fetch - let request: { readonly url: string; readonly init?: RequestInit } | undefined - globalThis.fetch = (async (input, init) => { - request = { url: String(input), init } - return new Response(JSON.stringify({ id: ORG }), { status: 200 }) - }) as typeof globalThis.fetch - - try { - const result = await Effect.runPromise( - updateCustomerBillingControls( - "am_sk_test", - "https://api.useautumn.com/", - ORG, - new UpdateBillingControlsRequest({ - spendLimits: [ - new UpdateBillingSpendLimit({ - featureId: "logs", - enabled: true, - limitType: "absolute", - overageLimit: 250, - }), - new UpdateBillingSpendLimit({ - featureId: "traces", - enabled: false, - }), - ], - usageAlerts: [ - new UpdateBillingUsageAlert({ - featureId: "logs", - enabled: true, - threshold: 80, - thresholdType: "usage_percentage", - name: "Maple billing warning", - }), - ], - }), - ), - ) +layer(FetchHttpClient.layer)("updateCustomerBillingControls", (it) => { + it.effect("uses Autumn's canonical customer update route and v2.3 wire shape", () => + Effect.gen(function* () { + let request: { readonly url: string; readonly init?: RequestInit } | undefined + const fetchImpl = (async (input, init) => { + request = { url: String(input), init } + return new Response(JSON.stringify({ id: ORG }), { status: 200 }) + }) as typeof globalThis.fetch + const httpClient = yield* HttpClient.HttpClient + + const result = yield* updateCustomerBillingControls( + httpClient, + "am_sk_test", + "https://api.useautumn.com/", + ORG, + new UpdateBillingControlsRequest({ + spendLimits: [ + new UpdateBillingSpendLimit({ + featureId: "logs", + enabled: true, + limitType: "absolute", + overageLimit: 250, + }), + new UpdateBillingSpendLimit({ + featureId: "traces", + enabled: false, + }), + ], + usageAlerts: [ + new UpdateBillingUsageAlert({ + featureId: "logs", + enabled: true, + threshold: 80, + thresholdType: "usage_percentage", + name: "Maple billing warning", + }), + ], + }), + ).pipe(Effect.provideService(FetchHttpClient.Fetch, fetchImpl)) assert.strictEqual(result.statusCode, 200) assert.strictEqual(request?.url, "https://api.useautumn.com/v1/customers.update") @@ -92,7 +92,7 @@ describe("updateCustomerBillingControls", () => { assert.strictEqual(headers.get("authorization"), "Bearer am_sk_test") assert.strictEqual(headers.get("content-type"), "application/json") assert.strictEqual(headers.get("x-api-version"), "2.3.0") - const requestBody = await new Response(request?.init?.body).text() + const requestBody = yield* Effect.promise(() => new Response(request?.init?.body).text()) assert.deepStrictEqual(JSON.parse(requestBody), { customer_id: ORG, billing_controls: { @@ -119,10 +119,8 @@ describe("updateCustomerBillingControls", () => { ], }, }) - } finally { - globalThis.fetch = originalFetch - } - }) + }), + ) }) describe("readCustomerCached", () => { diff --git a/apps/api/src/routes/v1/billing.http.ts b/apps/api/src/routes/v1/billing.http.ts index 8705e8e92..b2dc12a89 100644 --- a/apps/api/src/routes/v1/billing.http.ts +++ b/apps/api/src/routes/v1/billing.http.ts @@ -1,6 +1,6 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" -import { HttpServerRequest } from "effect/unstable/http" -import { Clock, Effect, Option, Redacted, Schema } from "effect" +import { FetchHttpClient, HttpClient, HttpServerRequest } from "effect/unstable/http" +import { Clock, Effect, Layer, Option, Redacted, Schema } from "effect" import type { CustomerData } from "autumn-js/backend" import { EdgeCacheService } from "@maple/cache" import { @@ -95,6 +95,7 @@ export const HttpBillingLive = HttpApiBuilder.group(MapleApi, "billing", (handle Effect.gen(function* () { const env = yield* Env const auth = yield* AuthService + const httpClient = yield* HttpClient.HttpClient const edgeCache = yield* EdgeCacheService const dailySpend = yield* DailySpendService const secretKey = Option.match(env.AUTUMN_SECRET_KEY, { @@ -201,6 +202,7 @@ export const HttpBillingLive = HttpApiBuilder.group(MapleApi, "billing", (handle ) yield* Effect.annotateCurrentSpan({ orgId: tenant.orgId }) const result = yield* updateCustomerBillingControls( + httpClient, secretKey, env.AUTUMN_API_URL, tenant.orgId, @@ -260,7 +262,7 @@ export const HttpBillingLive = HttpApiBuilder.group(MapleApi, "billing", (handle ) ) }), -) +).pipe(Layer.provide(FetchHttpClient.layer)) export const HttpBillingPublicLive = HttpApiBuilder.group(MapleApi, "billingPublic", (handlers) => Effect.gen(function* () { diff --git a/apps/api/src/routes/v1/observability.http.ts b/apps/api/src/routes/v1/observability.http.ts index 502e01788..545b13da2 100644 --- a/apps/api/src/routes/v1/observability.http.ts +++ b/apps/api/src/routes/v1/observability.http.ts @@ -9,7 +9,7 @@ import { diagnoseService, searchLogs, } from "@maple/query-engine/observability" -import { makeWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" +import { provideWarehouseExecutorFromTenant } from "@/services/warehouse/WarehouseQueryService" // Warehouse errors propagate with their canonical per-tag HTTP statuses (503 // transient, 429 quota, 400 validation, 502 otherwise) — declared on the @@ -31,7 +31,7 @@ export const HttpObservabilityLive = HttpApiBuilder.group(MapleApi, "observabili Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const services = yield* listServices(payload).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), ) return { services: [...services] } }), @@ -40,7 +40,7 @@ export const HttpObservabilityLive = HttpApiBuilder.group(MapleApi, "observabili Effect.gen(function* () { const tenant = yield* CurrentTenant.Context return yield* searchTraces(payload).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), Effect.map((r) => ({ ...r, spans: [...r.spans].map((s) => ({ ...s, attributes: { ...s.attributes } })), @@ -52,7 +52,7 @@ export const HttpObservabilityLive = HttpApiBuilder.group(MapleApi, "observabili Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const result = yield* inspectTrace(payload.traceId).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), ) return { traceId: result.traceId, @@ -67,9 +67,7 @@ export const HttpObservabilityLive = HttpApiBuilder.group(MapleApi, "observabili .handle("findErrors", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const errors = yield* findErrors(payload).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), - ) + const errors = yield* findErrors(payload).pipe(provideWarehouseExecutorFromTenant(tenant)) return { errors: [...errors] } }), ) @@ -77,7 +75,7 @@ export const HttpObservabilityLive = HttpApiBuilder.group(MapleApi, "observabili Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const result = yield* diagnoseService(payload).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), + provideWarehouseExecutorFromTenant(tenant), ) return { serviceName: result.serviceName, @@ -95,9 +93,7 @@ export const HttpObservabilityLive = HttpApiBuilder.group(MapleApi, "observabili .handle("searchLogs", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const result = yield* searchLogs(payload).pipe( - Effect.provide(makeWarehouseExecutorFromTenant(tenant)), - ) + const result = yield* searchLogs(payload).pipe(provideWarehouseExecutorFromTenant(tenant)) return { timeRange: result.timeRange, total: result.total, diff --git a/apps/api/src/services/alerts/AlertDeliveryDispatch.ts b/apps/api/src/services/alerts/AlertDeliveryDispatch.ts index 5ce5abd54..d99514588 100644 --- a/apps/api/src/services/alerts/AlertDeliveryDispatch.ts +++ b/apps/api/src/services/alerts/AlertDeliveryDispatch.ts @@ -531,358 +531,346 @@ export const dispatchDelivery = ( chatUrl: string, deps: DispatchDeps, ): Effect.Effect => - Effect.gen(function* () { - return yield* Match.value(context.secretConfig).pipe( - Match.discriminatorsExhaustive("type")({ - "slack-bot": (config) => - Effect.gen(function* () { - const botToken = yield* deps.resolveSlackBotToken(context.destination.orgId) - const templated = renderTitleBody(context, "slack-bot", linkUrl, chatUrl) - const blocks = templated - ? buildSlackBlocksFromTemplate( - templated.title, - templated.body, - context, - linkUrl, - chatUrl, - ) - : buildSlackBlocks(context, linkUrl, chatUrl) - const response = yield* runTimedFetch("slack-bot", "Slack", fetchFn, timeoutMs, () => - fetchFn("https://slack.com/api/chat.postMessage", { - method: "POST", - headers: { - "content-type": "application/json; charset=utf-8", - authorization: `Bearer ${botToken}`, - }, - body: JSON.stringify({ - channel: config.channelId, - // Blocks ride inside a colored attachment so the message - // carries the severity color bar — same as the webhook - // destination (the bar has no Block Kit equivalent). No - // top-level `text`: alongside attachments Slack renders it - // as a duplicate line above the bar; `fallback` carries - // the notification-preview one-liner instead. - attachments: [ - { - color: slackAttachmentColor(context.eventType, context.severity), - fallback: templated?.title ?? buildSlackFallbackText(context), - blocks, - }, - ], - }), + Match.value(context.secretConfig).pipe( + Match.discriminatorsExhaustive("type")({ + "slack-bot": (config) => + Effect.gen(function* () { + const botToken = yield* deps.resolveSlackBotToken(context.destination.orgId) + const templated = renderTitleBody(context, "slack-bot", linkUrl, chatUrl) + const blocks = templated + ? buildSlackBlocksFromTemplate( + templated.title, + templated.body, + context, + linkUrl, + chatUrl, + ) + : buildSlackBlocks(context, linkUrl, chatUrl) + const response = yield* runTimedFetch("slack-bot", "Slack", fetchFn, timeoutMs, () => + fetchFn("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "content-type": "application/json; charset=utf-8", + authorization: `Bearer ${botToken}`, + }, + body: JSON.stringify({ + channel: config.channelId, + // Blocks ride inside a colored attachment so the message + // carries the severity color bar — same as the webhook + // destination (the bar has no Block Kit equivalent). No + // top-level `text`: alongside attachments Slack renders it + // as a duplicate line above the bar; `fallback` carries + // the notification-preview one-liner instead. + attachments: [ + { + color: slackAttachmentColor(context.eventType, context.severity), + fallback: templated?.title ?? buildSlackFallbackText(context), + blocks, + }, + ], }), + }), + ) + if (!response.ok) { + const detail = yield* readErrorBody(response) + return yield* Effect.fail( + makeDeliveryError( + `Slack delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, + "slack-bot", + ), ) - if (!response.ok) { - const detail = yield* readErrorBody(response) - return yield* Effect.fail( - makeDeliveryError( - `Slack delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, - "slack-bot", - ), - ) - } - // Slack Web API returns HTTP 200 with `{ ok: false, error }` on - // logical failures, so the JSON body — not the status — is the - // source of truth. - const rawPayload = yield* Effect.tryPromise({ - try: () => response.json(), - catch: (error) => - makeDeliveryError("Slack returned a non-JSON response", "slack-bot", error), + } + // Slack Web API returns HTTP 200 with `{ ok: false, error }` on + // logical failures, so the JSON body — not the status — is the + // source of truth. + const rawPayload = yield* Effect.tryPromise({ + try: () => response.json(), + catch: (error) => + makeDeliveryError("Slack returned a non-JSON response", "slack-bot", error), + }) + const payload = yield* decodeSlackPostMessageResponse(rawPayload).pipe( + Effect.mapError((error) => + makeDeliveryError( + `Slack returned an unexpected response payload: ${error.message}`, + "slack-bot", + ), + ), + ) + if (!payload.ok) { + const error = payload.error ?? "unknown_error" + // HTTP 200 + `ok:false` is the dominant operational failure here + // (`not_in_channel` after a rename/kick). NotificationDispatcher + // only records outcome "failed", so annotate the Slack error code + // on the span to make it aggregatable. + yield* Effect.annotateCurrentSpan({ + "maple.delivery.destination_type": "slack-bot", + "maple.delivery.provider_error": error, }) - const payload = yield* decodeSlackPostMessageResponse(rawPayload).pipe( - Effect.mapError((error) => - makeDeliveryError( - `Slack returned an unexpected response payload: ${error.message}`, - "slack-bot", - ), + const message = + error === "not_in_channel" || error === "channel_not_found" + ? `Slack rejected the message (${error}) — invite the Maple bot to the channel and try again` + : `Slack rejected the message: ${error}` + return yield* Effect.fail(makeDeliveryError(message, "slack-bot")) + } + return { + providerMessage: `Delivered to Slack #${config.channelName ?? config.channelId}`, + providerReference: payload.ts ?? null, + responseCode: response.status, + } as DispatchResult + }), + pagerduty: (config) => + Effect.gen(function* () { + const templated = renderTitleBody(context, "pagerduty", linkUrl, chatUrl) + const body = { + routing_key: config.integrationKey, + event_action: context.eventType === "resolve" ? "resolve" : "trigger", + dedup_key: context.dedupeKey, + payload: { + summary: truncate( + templated?.title ?? `${context.ruleName} ${context.eventType}`, + 1024, ), - ) - if (!payload.ok) { - const error = payload.error ?? "unknown_error" - // HTTP 200 + `ok:false` is the dominant operational failure here - // (`not_in_channel` after a rename/kick). NotificationDispatcher - // only records outcome "failed", so annotate the Slack error code - // on the span to make it aggregatable. - yield* Effect.annotateCurrentSpan({ - "maple.delivery.destination_type": "slack-bot", - "maple.delivery.provider_error": error, - }) - const message = - error === "not_in_channel" || error === "channel_not_found" - ? `Slack rejected the message (${error}) — invite the Maple bot to the channel and try again` - : `Slack rejected the message: ${error}` - return yield* Effect.fail(makeDeliveryError(message, "slack-bot")) - } - return { - providerMessage: `Delivered to Slack #${config.channelName ?? config.channelId}`, - providerReference: payload.ts ?? null, - responseCode: response.status, - } as DispatchResult - }), - pagerduty: (config) => - Effect.gen(function* () { - const templated = renderTitleBody(context, "pagerduty", linkUrl, chatUrl) - const body = { - routing_key: config.integrationKey, - event_action: context.eventType === "resolve" ? "resolve" : "trigger", - dedup_key: context.dedupeKey, - payload: { - summary: truncate( - templated?.title ?? `${context.ruleName} ${context.eventType}`, - 1024, - ), - source: context.groupKey ?? "maple-alerts", - severity: context.severity === "critical" ? "critical" : "warning", - custom_details: { - ...(templated ? { message: templated.body } : {}), - ruleName: context.ruleName, - signalType: context.signalType, - value: context.value, - threshold: context.threshold, - thresholdUpper: context.thresholdUpper, - comparator: context.comparator, - groupKey: context.groupKey, - linkUrl, - chatUrl, - }, - }, - links: [ - { href: linkUrl, text: "Open in Maple" }, - { href: chatUrl, text: "Ask Maple AI" }, - ], - } - const response = yield* runTimedFetch( - "pagerduty", - "PagerDuty", - fetchFn, - timeoutMs, - () => - fetchFn("https://events.pagerduty.com/v2/enqueue", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - }), - ) - if (!response.ok) { - const detail = yield* readErrorBody(response) - return yield* Effect.fail( - makeDeliveryError( - `PagerDuty delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, - "pagerduty", - ), - ) - } - return { - providerMessage: "Delivered to PagerDuty", - providerReference: context.dedupeKey, - responseCode: response.status, - } as DispatchResult - }), - webhook: (config) => - Effect.gen(function* () { - const headers: Record = { - "content-type": "application/json", - "x-maple-event-type": context.eventType, - "x-maple-delivery-key": context.deliveryKey, - } - if (config.signingSecret) { - headers["x-maple-signature"] = createHmac("sha256", config.signingSecret) - .update(payloadJson) - .digest("hex") - } - const response = yield* runTimedFetch("webhook", "Webhook", fetchFn, timeoutMs, () => - safeFetch(config.url, { method: "POST", headers, body: payloadJson, fetchFn }), - ) - if (!response.ok) { - const detail = yield* readErrorBody(response) - return yield* Effect.fail( - makeDeliveryError( - `Webhook delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, - "webhook", - ), - ) - } - return { - providerMessage: "Delivered to webhook", - providerReference: context.dedupeKey, - responseCode: response.status, - } as DispatchResult - }), - "hazel-oauth": (config) => - Effect.gen(function* () { - // Hazel exposes per-integration sibling endpoints under the same - // `:webhookId/:token` prefix (see hazel - // packages/domain/src/http/incoming-webhooks.ts). The stored - // webhookUrl is the base; we append `/maple` to hit the - // `executeMaple` handler — without it, the payload routes to the - // Discord-style `execute` endpoint and is rejected. - const hazelUrl = `${config.webhookUrl.replace(/\/$/, "")}/maple` - const incidentStatus = context.eventType === "resolve" ? "resolved" : "open" - const body = JSON.stringify({ - eventType: context.eventType, - incidentId: context.incidentId, - incidentStatus, - dedupeKey: context.dedupeKey, - rule: { - id: context.ruleId, - name: context.ruleName, + source: context.groupKey ?? "maple-alerts", + severity: context.severity === "critical" ? "critical" : "warning", + custom_details: { + ...(templated ? { message: templated.body } : {}), + ruleName: context.ruleName, signalType: context.signalType, - severity: context.severity, - groupKey: context.groupKey, - comparator: context.comparator, + value: context.value, threshold: context.threshold, - windowMinutes: context.windowMinutes, + thresholdUpper: context.thresholdUpper, + comparator: context.comparator, + groupKey: context.groupKey, + linkUrl, + chatUrl, }, - observed: { - value: context.value, - sampleCount: context.sampleCount, + }, + links: [ + { href: linkUrl, text: "Open in Maple" }, + { href: chatUrl, text: "Ask Maple AI" }, + ], + } + const response = yield* runTimedFetch("pagerduty", "PagerDuty", fetchFn, timeoutMs, () => + fetchFn("https://events.pagerduty.com/v2/enqueue", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ) + if (!response.ok) { + const detail = yield* readErrorBody(response) + return yield* Effect.fail( + makeDeliveryError( + `PagerDuty delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, + "pagerduty", + ), + ) + } + return { + providerMessage: "Delivered to PagerDuty", + providerReference: context.dedupeKey, + responseCode: response.status, + } as DispatchResult + }), + webhook: (config) => + Effect.gen(function* () { + const headers: Record = { + "content-type": "application/json", + "x-maple-event-type": context.eventType, + "x-maple-delivery-key": context.deliveryKey, + } + if (config.signingSecret) { + headers["x-maple-signature"] = createHmac("sha256", config.signingSecret) + .update(payloadJson) + .digest("hex") + } + const response = yield* runTimedFetch("webhook", "Webhook", fetchFn, timeoutMs, () => + safeFetch(config.url, { method: "POST", headers, body: payloadJson, fetchFn }), + ) + if (!response.ok) { + const detail = yield* readErrorBody(response) + return yield* Effect.fail( + makeDeliveryError( + `Webhook delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, + "webhook", + ), + ) + } + return { + providerMessage: "Delivered to webhook", + providerReference: context.dedupeKey, + responseCode: response.status, + } as DispatchResult + }), + "hazel-oauth": (config) => + Effect.gen(function* () { + // Hazel exposes per-integration sibling endpoints under the same + // `:webhookId/:token` prefix (see hazel + // packages/domain/src/http/incoming-webhooks.ts). The stored + // webhookUrl is the base; we append `/maple` to hit the + // `executeMaple` handler — without it, the payload routes to the + // Discord-style `execute` endpoint and is rejected. + const hazelUrl = `${config.webhookUrl.replace(/\/$/, "")}/maple` + const incidentStatus = context.eventType === "resolve" ? "resolved" : "open" + const body = JSON.stringify({ + eventType: context.eventType, + incidentId: context.incidentId, + incidentStatus, + dedupeKey: context.dedupeKey, + rule: { + id: context.ruleId, + name: context.ruleName, + signalType: context.signalType, + severity: context.severity, + groupKey: context.groupKey, + comparator: context.comparator, + threshold: context.threshold, + windowMinutes: context.windowMinutes, + }, + observed: { + value: context.value, + sampleCount: context.sampleCount, + }, + template: context.template ?? null, + linkUrl, + chatUrl, + sentAt: new Date(yield* Clock.currentTimeMillis).toISOString(), + }) + const response = yield* runTimedFetch("hazel-oauth", "Hazel", fetchFn, timeoutMs, () => + safeFetch(hazelUrl, { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-type": context.eventType, + "x-maple-delivery-key": context.deliveryKey, }, - template: context.template ?? null, - linkUrl, - chatUrl, - sentAt: new Date(yield* Clock.currentTimeMillis).toISOString(), - }) - const response = yield* runTimedFetch( - "hazel-oauth", - "Hazel", + body, fetchFn, - timeoutMs, - () => - safeFetch(hazelUrl, { - method: "POST", - headers: { - "content-type": "application/json", - "x-maple-event-type": context.eventType, - "x-maple-delivery-key": context.deliveryKey, - }, - body, - fetchFn, - }), + }), + ) + if (response.status === 401 || response.status === 403) { + return yield* Effect.fail( + makeDeliveryError( + "Hazel rejected the webhook token — reconfigure the channel", + "hazel-oauth", + ), ) - if (response.status === 401 || response.status === 403) { - return yield* Effect.fail( - makeDeliveryError( - "Hazel rejected the webhook token — reconfigure the channel", - "hazel-oauth", - ), - ) - } - if (response.status === 404) { - return yield* Effect.fail( - makeDeliveryError( - "Hazel webhook no longer exists — pick a different channel", - "hazel-oauth", - ), - ) - } - if (!response.ok) { - const detail = yield* readErrorBody(response) - return yield* Effect.fail( - makeDeliveryError( - `Hazel delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, - "hazel-oauth", - ), + } + if (response.status === 404) { + return yield* Effect.fail( + makeDeliveryError( + "Hazel webhook no longer exists — pick a different channel", + "hazel-oauth", + ), + ) + } + if (!response.ok) { + const detail = yield* readErrorBody(response) + return yield* Effect.fail( + makeDeliveryError( + `Hazel delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, + "hazel-oauth", + ), + ) + } + return { + providerMessage: `Delivered to Hazel #${config.hazelChannelName}`, + providerReference: context.dedupeKey, + responseCode: response.status, + } as DispatchResult + }), + discord: (config) => + Effect.gen(function* () { + const templated = renderTitleBody(context, "discord", linkUrl, chatUrl) + const embeds = templated + ? buildDiscordEmbedsFromTemplate( + templated.title, + templated.body, + context, + linkUrl, + chatUrl, ) - } - return { - providerMessage: `Delivered to Hazel #${config.hazelChannelName}`, - providerReference: context.dedupeKey, - responseCode: response.status, - } as DispatchResult - }), - discord: (config) => - Effect.gen(function* () { - const templated = renderTitleBody(context, "discord", linkUrl, chatUrl) - const embeds = templated - ? buildDiscordEmbedsFromTemplate( - templated.title, - templated.body, - context, - linkUrl, - chatUrl, - ) - : buildDiscordEmbeds(context, linkUrl, chatUrl) - const response = yield* runTimedFetch("discord", "Discord", fetchFn, timeoutMs, () => - safeFetch(config.webhookUrl, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - username: "Maple Alerts", - content: - templated?.title ?? - `**${context.ruleName}**: ${formatEventTypeLabel(context.eventType)}`, - embeds, - }), - fetchFn, + : buildDiscordEmbeds(context, linkUrl, chatUrl) + const response = yield* runTimedFetch("discord", "Discord", fetchFn, timeoutMs, () => + safeFetch(config.webhookUrl, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + username: "Maple Alerts", + content: + templated?.title ?? + `**${context.ruleName}**: ${formatEventTypeLabel(context.eventType)}`, + embeds, }), + fetchFn, + }), + ) + if (!response.ok) { + const detail = yield* readErrorBody(response) + return yield* Effect.fail( + makeDeliveryError( + `Discord delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, + "discord", + ), ) - if (!response.ok) { - const detail = yield* readErrorBody(response) - return yield* Effect.fail( - makeDeliveryError( - `Discord delivery failed with ${response.status}${detail ? `: ${detail}` : ""}`, - "discord", - ), - ) - } - return { - providerMessage: "Delivered to Discord", - providerReference: null, - responseCode: response.status, - } as DispatchResult - }), - email: (config) => - Effect.gen(function* () { - const { subject, html } = yield* buildAlertEmailContent(context, linkUrl, chatUrl) - const outcomes = yield* Effect.forEach(config.members, (member) => - deps.sendEmail(member.email, subject, html).pipe( - Effect.match({ - onSuccess: () => ({ member, error: null as string | null }), - onFailure: (error) => ({ member, error: error.message }), - }), + } + return { + providerMessage: "Delivered to Discord", + providerReference: null, + responseCode: response.status, + } as DispatchResult + }), + email: (config) => + Effect.gen(function* () { + const { subject, html } = yield* buildAlertEmailContent(context, linkUrl, chatUrl) + const outcomes = yield* Effect.forEach(config.members, (member) => + deps.sendEmail(member.email, subject, html).pipe( + Effect.match({ + onSuccess: () => ({ member, error: null as string | null }), + onFailure: (error) => ({ member, error: error.message }), + }), + ), + ) + const failures = outcomes.filter((o) => o.error != null) + const count = config.members.length + + if (failures.length === count && count > 0) { + // Nobody received the email — safe to fail retryable; the retry + // re-sends to members who all got nothing. Keep the first failure + // message verbatim so timeout classification survives aggregation. + return yield* Effect.fail( + makeDeliveryError( + `Email delivery failed for all ${count} member${count === 1 ? "" : "s"}: ${failures[0]!.error}`, + "email", ), ) - const failures = outcomes.filter((o) => o.error != null) - const count = config.members.length - - if (failures.length === count && count > 0) { - // Nobody received the email — safe to fail retryable; the retry - // re-sends to members who all got nothing. Keep the first failure - // message verbatim so timeout classification survives aggregation. - return yield* Effect.fail( - makeDeliveryError( - `Email delivery failed for all ${count} member${count === 1 ? "" : "s"}: ${failures[0]!.error}`, - "email", - ), - ) - } - - if (failures.length > 0) { - // Partial success is terminal: there is no per-member attempt - // state, so retrying the event would re-email the members who - // already received it. Report success and surface the failures. - yield* Effect.logWarning("Alert email delivered to a subset of members").pipe( - Effect.annotateLogs({ - failedCount: failures.length, - memberCount: count, - firstError: failures[0]!.error, - }), - ) - return { - providerMessage: `Emailed ${count - failures.length} of ${count} members; failed: ${failures - .map((f) => `${f.member.email} (${f.error})`) - .join(", ")}`, - providerReference: null, - responseCode: null, - } as DispatchResult - } - + } + + if (failures.length > 0) { + // Partial success is terminal: there is no per-member attempt + // state, so retrying the event would re-email the members who + // already received it. Report success and surface the failures. + yield* Effect.logWarning("Alert email delivered to a subset of members").pipe( + Effect.annotateLogs({ + failedCount: failures.length, + memberCount: count, + firstError: failures[0]!.error, + }), + ) return { - providerMessage: `Emailed ${count} member${count === 1 ? "" : "s"}`, + providerMessage: `Emailed ${count - failures.length} of ${count} members; failed: ${failures + .map((f) => `${f.member.email} (${f.error})`) + .join(", ")}`, providerReference: null, responseCode: null, } as DispatchResult - }), - }), - ) - }) + } + + return { + providerMessage: `Emailed ${count} member${count === 1 ? "" : "s"}`, + providerReference: null, + responseCode: null, + } as DispatchResult + }), + }), + ) diff --git a/apps/api/src/services/auth/auth.test.ts b/apps/api/src/services/auth/auth.test.ts index 3d50d2bb7..fd4649082 100644 --- a/apps/api/src/services/auth/auth.test.ts +++ b/apps/api/src/services/auth/auth.test.ts @@ -3,11 +3,12 @@ import { Effect, Schema } from "effect" import { RoleName } from "@maple/domain/http" import { isAdmin, requireAdmin } from "./auth" -const role = (raw: string) => Schema.decodeUnknownSync(RoleName)(raw) +const role = (raw: string) => Schema.decodeSync(RoleName)(raw) -class TestForbiddenError extends Error { - readonly _tag = "TestForbiddenError" -} +class TestForbiddenError extends Schema.TaggedError()( + "@maple/api/test/TestForbiddenError", + { message: Schema.String }, +) {} describe("isAdmin", () => { it("returns true for root", () => { @@ -29,13 +30,13 @@ describe("isAdmin", () => { describe("requireAdmin", () => { it.effect("succeeds when at least one role is admin", () => - requireAdmin([role("root")], () => new TestForbiddenError("nope")), + requireAdmin([role("root")], () => new TestForbiddenError({ message: "nope" })), ) it.effect("fails with the supplied error for non-admin roles", () => Effect.gen(function* () { const error = yield* Effect.flip( - requireAdmin([role("org:member")], () => new TestForbiddenError("nope")), + requireAdmin([role("org:member")], () => new TestForbiddenError({ message: "nope" })), ) assert.instanceOf(error, TestForbiddenError) }), diff --git a/apps/api/src/services/billing/autumn-client.ts b/apps/api/src/services/billing/autumn-client.ts index bc080b5f4..4e444da46 100644 --- a/apps/api/src/services/billing/autumn-client.ts +++ b/apps/api/src/services/billing/autumn-client.ts @@ -1,5 +1,5 @@ import { Data, Effect, Schema } from "effect" -import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" import type { autumnHandler, CustomerData } from "autumn-js/backend" import type { EdgeCacheServiceShape } from "@maple/cache" import { isActivePlanSubscription } from "@maple/domain/billing" @@ -128,6 +128,7 @@ const toBillingUpstreamError = (error: unknown) => * billing controls live on the canonical REST surface instead. */ export const updateCustomerBillingControls = ( + client: HttpClient.HttpClient, secretKey: string | undefined, apiUrl: string, orgId: string, @@ -136,7 +137,6 @@ export const updateCustomerBillingControls = ( secretKey === undefined ? Effect.fail(new BillingUpstreamError({ message: "Billing is not configured" })) : Effect.gen(function* () { - const client = yield* HttpClient.HttpClient const request = yield* HttpClientRequest.bodyJson( HttpClientRequest.post(`${apiUrl.replace(/\/+$/, "")}/v1/customers.update`, { headers: { @@ -162,17 +162,15 @@ export const updateCustomerBillingControls = ( })), }, }, - ).pipe(Effect.mapError(toBillingUpstreamError)) - const response = yield* client.execute(request).pipe(Effect.mapError(toBillingUpstreamError)) - const text = yield* response.text.pipe(Effect.mapError(toBillingUpstreamError)) + ) + const response = yield* client.execute(request) + const text = yield* response.text const responseBody = text.length === 0 ? {} - : yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(text).pipe( - Effect.mapError(toBillingUpstreamError), - ) + : yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(text) return { statusCode: response.status, response: responseBody } as AutumnResult - }).pipe(Effect.provide(FetchHttpClient.layer)) + }).pipe(Effect.mapError(toBillingUpstreamError)) // Surface a readable message for a non-2xx Autumn response (it carries a // `{ message }` / `{ error }` body) so the client error isn't an opaque 502. diff --git a/apps/api/src/services/errors/error-tick-persistence.ts b/apps/api/src/services/errors/error-tick-persistence.ts index af983aaf4..66052a072 100644 --- a/apps/api/src/services/errors/error-tick-persistence.ts +++ b/apps/api/src/services/errors/error-tick-persistence.ts @@ -92,6 +92,8 @@ export const isErrorTickClaimLost = (error: { readonly message: string }): boole * worker stalled long enough for the crash-recovery TTL to elapse and another * invocation took over, so the window must roll back rather than double-apply. */ +// Database.execute rewraps transaction throws, so the marker-bearing native message is the intentional boundary. +// oxlint-disable-next-line effecttsgo/extends-native-error export class ErrorTickClaimLost extends Error { readonly _tag = "ErrorTickClaimLost" constructor( @@ -103,8 +105,6 @@ export class ErrorTickClaimLost extends Error { } } -type TickTransaction = Parameters[0]>[0] - /** * The scan groups by fingerprint, so duplicates are not expected — but a * multi-row `ON CONFLICT DO UPDATE` errors outright if the same conflict target diff --git a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts index c3f29b987..17c68b8b0 100644 --- a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts +++ b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts @@ -1005,6 +1005,7 @@ export class CloudflareAnalyticsService extends Context.Service< make: Effect.gen(function* () { const database = yield* Database const env = yield* Env + const httpClient = yield* HttpClient.HttpClient const warehouse = yield* WarehouseQueryService const oauth = yield* CloudflareOAuthService const ingestKeys = yield* OrgIngestKeysService @@ -1403,11 +1404,10 @@ export class CloudflareAnalyticsService extends Context.Service< const total = rows.sumRows.length + rows.gaugeRows.length if (total === 0) return 0 const payload = metricRowsToOtlp(rows.sumRows, rows.gaugeRows) - const client = yield* HttpClient.HttpClient const request = HttpClientRequest.post(ingestMetricsUrl, { headers: { authorization: `Bearer ${ingestKey}`, "content-type": "application/json" }, }).pipe(HttpClientRequest.bodyJsonUnsafe(payload)) - const response = yield* client + const response = yield* httpClient .execute(request) .pipe(Effect.annotateSpans("peer.service", "ingest")) if (response.status >= 300) { @@ -1423,7 +1423,6 @@ export class CloudflareAnalyticsService extends Context.Service< }, (effect) => effect.pipe( - Effect.provide(FetchHttpClient.layer), Effect.mapError((error) => error instanceof IntegrationsUpstreamError ? error @@ -2403,5 +2402,5 @@ export class CloudflareAnalyticsService extends Context.Service< } satisfies CloudflareAnalyticsServiceShape }), }) { - static readonly layer = Layer.effect(this, this.make) + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(FetchHttpClient.layer)) } diff --git a/apps/api/src/services/integrations/CloudflareApiImpl.ts b/apps/api/src/services/integrations/CloudflareApiImpl.ts index 9cb6ac761..16627764d 100644 --- a/apps/api/src/services/integrations/CloudflareApiImpl.ts +++ b/apps/api/src/services/integrations/CloudflareApiImpl.ts @@ -93,7 +93,13 @@ const runWithToken = ( accessToken: string, effect: Effect.Effect, apiBaseUrl?: string, -): Effect.Effect => effect.pipe(Effect.provide(runtimeLayer(accessToken, apiBaseUrl))) +): Effect.Effect => + effect.pipe( + // The token and API base are per invocation, so this cannot be hoisted into + // the static service graph; the layer closes the distilled SDK operation. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + Effect.provide(runtimeLayer(accessToken, apiBaseUrl)), + ) /** Like {@link runWithToken} but collapses the distilled error union to a Maple domain error. */ const runMapped = ( diff --git a/apps/api/src/services/integrations/PlanetScaleConnectionService.ts b/apps/api/src/services/integrations/PlanetScaleConnectionService.ts index 081fcd7a6..0ae4a4216 100644 --- a/apps/api/src/services/integrations/PlanetScaleConnectionService.ts +++ b/apps/api/src/services/integrations/PlanetScaleConnectionService.ts @@ -118,7 +118,6 @@ const toPersistenceError = (error: unknown) => }) const decodeUserIdSync = Schema.decodeUnknownSync(UserId) -const decodeScrapeTargetIdSync = Schema.decodeUnknownSync(ScrapeTargetId) export class PlanetScaleConnectionService extends Context.Service< PlanetScaleConnectionService, @@ -353,7 +352,7 @@ export class PlanetScaleConnectionService extends Context.Service< detectedPermissions: connection.detectedPermissionsJson ?? null, scrapeTarget: target ? new PlanetScaleScrapeTargetSummary({ - id: decodeScrapeTargetIdSync(target.id), + id: target.id, enabled: target.enabled, scrapeIntervalSeconds: target.scrapeIntervalSeconds, includeBranches: discoveryConfig?.includeBranches ?? [], @@ -475,7 +474,7 @@ export class PlanetScaleConnectionService extends Context.Service< // targets carry no credentials — the scraper resolves the OAuth grant at // scrape time (authType "planetscale_oauth"). const adoptable = yield* findAdoptableTarget(orgId, organization) - let scrapeTargetId: string + let scrapeTargetId: ScrapeTargetId let createdTarget = false if (adoptable !== null) { // An adopted row with working service-token credentials keeps them — @@ -485,7 +484,7 @@ export class PlanetScaleConnectionService extends Context.Service< const keepsToken = adoptable.authType === "token" && adoptable.authCredentialsCiphertext !== null yield* scrapeTargetsService - .update(orgId, decodeScrapeTargetIdSync(adoptable.id), { + .update(orgId, adoptable.id, { ...(keepsToken ? {} : { authType: "planetscale_oauth" }), ...(request.includeBranches !== undefined ? { includeBranches: request.includeBranches } @@ -610,25 +609,23 @@ export class PlanetScaleConnectionService extends Context.Service< Effect.mapError(toPersistenceError), Effect.tapError(() => createdTarget - ? scrapeTargetsService - .delete(orgId, decodeScrapeTargetIdSync(scrapeTargetId)) - .pipe( - Effect.catchTag( - "@maple/http/errors/ScrapeTargetNotFoundError", - () => Effect.void, - ), - Effect.catch((error) => - Effect.logWarning( - "Failed to compensate newly created PlanetScale target after binding failure", - ).pipe( - Effect.annotateLogs({ - orgId, - scrapeTargetId, - error: String(error), - }), - ), + ? scrapeTargetsService.delete(orgId, scrapeTargetId).pipe( + Effect.catchTag( + "@maple/http/errors/ScrapeTargetNotFoundError", + () => Effect.void, + ), + Effect.catch((error) => + Effect.logWarning( + "Failed to compensate newly created PlanetScale target after binding failure", + ).pipe( + Effect.annotateLogs({ + orgId, + scrapeTargetId, + error: String(error), + }), ), - ) + ), + ) : Effect.void, ), ) @@ -688,7 +685,7 @@ export class PlanetScaleConnectionService extends Context.Service< ) } yield* scrapeTargetsService - .update(orgId, decodeScrapeTargetIdSync(target.id), { + .update(orgId, target.id, { authType: "token", authCredentials: JSON.stringify({ tokenId, tokenSecret: request.tokenSecret }), enabled: true, @@ -707,7 +704,7 @@ export class PlanetScaleConnectionService extends Context.Service< // owns it (a user-created row adopted by a *different* connection stays). const target = yield* selectManagedTarget(connection) if (target !== null && target.managedBy === managedByForConnection(connection.id)) { - yield* scrapeTargetsService.delete(orgId, decodeScrapeTargetIdSync(target.id)).pipe( + yield* scrapeTargetsService.delete(orgId, target.id).pipe( Effect.catchTag("@maple/http/errors/ScrapeTargetNotFoundError", () => Effect.annotateCurrentSpan("maple.planetscale.disconnect_target_missing", true), ), diff --git a/apps/api/src/services/integrations/scrape-check-retention.test.ts b/apps/api/src/services/integrations/scrape-check-retention.test.ts index 7930a03ee..668639833 100644 --- a/apps/api/src/services/integrations/scrape-check-retention.test.ts +++ b/apps/api/src/services/integrations/scrape-check-retention.test.ts @@ -1,8 +1,12 @@ import { assert, describe, it } from "@effect/vitest" +import { ScrapeTargetId } from "@maple/domain" +import { Schema } from "effect" import { canExceedRowCap, type RetentionTarget } from "@/services/integrations/scrape-check-retention" +const targetId = Schema.decodeSync(ScrapeTargetId)("11111111-1111-4111-8111-111111111111") + const target = (overrides: Partial = {}): RetentionTarget => ({ - id: "tgt_1", + id: targetId, targetType: "prometheus", scrapeIntervalSeconds: 15, ...overrides, diff --git a/apps/api/src/services/integrations/scrape-check-retention.ts b/apps/api/src/services/integrations/scrape-check-retention.ts index afb3d65bc..4e2d5eb30 100644 --- a/apps/api/src/services/integrations/scrape-check-retention.ts +++ b/apps/api/src/services/integrations/scrape-check-retention.ts @@ -1,7 +1,9 @@ +import type { ScrapeTargetId } from "@maple/domain" import { scrapeTargetChecks, scrapeTargets } from "@maple/db" import { and, desc, eq, inArray, lt } from "drizzle-orm" import { Clock, Effect } from "effect" import { Database } from "@/platform/DatabaseLive" +import { msToDate } from "@/platform/time" /** * Check-history retention for `scrape_target_checks`. @@ -23,7 +25,7 @@ const CHECK_MAX_ROWS_PER_TARGET = 10_000 /** What retention needs to know about a target to prune its check history. */ export interface RetentionTarget { - readonly id: string + readonly id: ScrapeTargetId readonly targetType: string readonly scrapeIntervalSeconds: number } @@ -60,7 +62,7 @@ export const pruneChecksForTargets = Effect.fn("ScrapeCheckRetention.pruneForTar ) { if (targets.length === 0) return const now = yield* Clock.currentTimeMillis - const cutoff = new Date(now - CHECK_RETENTION_MS) + const cutoff = msToDate(now - CHECK_RETENTION_MS) const ids = targets.map((target) => target.id) const capCandidates = targets.filter(canExceedRowCap) const database = yield* Database diff --git a/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts b/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts index 13a7004d8..c5be6625a 100644 --- a/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts +++ b/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "@effect/vitest" +import { afterEach, describe, expect, it, layer } from "@effect/vitest" import { OrgClickHouseSettingsUpstreamRejectedError, OrgClickHouseSettingsUpstreamUnavailableError, @@ -158,7 +158,7 @@ describe("isRetryableUpstream", () => { }) }) -describe("execClickHouse", () => { +layer(FetchHttpClient.layer, { excludeTestServices: true })("execClickHouse", (it) => { it.effect("uses manual redirects and rejects every 3xx without following it", () => Effect.gen(function* () { let redirectMode: RequestRedirect | undefined @@ -180,7 +180,7 @@ describe("execClickHouse", () => { }), ) - it.live("maps a Cloudflare 524 to a clear, actionable message (and retries 52x)", () => + it.effect("maps a Cloudflare 524 to a clear, actionable message (and retries 52x)", () => Effect.gen(function* () { const { state, fetchImpl } = makeFetch(() => Promise.resolve(mockResponse("error code: 524", 524)), @@ -199,7 +199,7 @@ describe("execClickHouse", () => { }), ) - it.live("retries a transient 503 then succeeds", () => + it.effect("retries a transient 503 then succeeds", () => Effect.gen(function* () { const { state, fetchImpl } = makeFetch(() => Promise.resolve( diff --git a/apps/api/src/services/org/OrgClickHouseSettingsService.ts b/apps/api/src/services/org/OrgClickHouseSettingsService.ts index 2b6f84c65..126529b48 100644 --- a/apps/api/src/services/org/OrgClickHouseSettingsService.ts +++ b/apps/api/src/services/org/OrgClickHouseSettingsService.ts @@ -778,9 +778,8 @@ const mapStatusToError = ( ) } -export const execClickHouse = (config: ClickHouseExecConfig, sql: string) => +const execClickHouseWithClient = (client: HttpClient.HttpClient, config: ClickHouseExecConfig, sql: string) => Effect.gen(function* () { - const client = yield* HttpClient.HttpClient const request = HttpClientRequest.post(buildClickHouseUrl(config), { headers: buildClickHouseHeaders(config), }).pipe(HttpClientRequest.bodyText(sql)) @@ -829,9 +828,11 @@ export const execClickHouse = (config: ClickHouseExecConfig, sql: string) => ), }), Effect.retry({ schedule: CLICKHOUSE_RETRY_SCHEDULE, while: isRetryableUpstream }), - Effect.provide(FetchHttpClient.layer), ) +export const execClickHouse = (config: ClickHouseExecConfig, sql: string) => + HttpClient.HttpClient.use((client) => execClickHouseWithClient(client, config, sql)) + interface ClickHouseTableRow { readonly name: string readonly engine: string @@ -842,15 +843,15 @@ interface ClickHouseColumnRow { readonly type: string } -const fetchActualSchema = (config: ClickHouseExecConfig) => +const fetchActualSchema = (client: HttpClient.HttpClient, config: ClickHouseExecConfig) => Effect.gen(function* () { // Tables: name + engine. Engine="MaterializedView" → MV; everything else → table. const tablesSql = `SELECT name, engine FROM system.tables WHERE database = '${config.database.replace(/'/g, "''")}' FORMAT JSONEachRow` - const tablesText = yield* execClickHouse(config, tablesSql) + const tablesText = yield* execClickHouseWithClient(client, config, tablesSql) const tableRows = parseJsonEachRow(tablesText) const columnsSql = `SELECT table, name, type FROM system.columns WHERE database = '${config.database.replace(/'/g, "''")}' FORMAT JSONEachRow` - const columnsText = yield* execClickHouse(config, columnsSql) + const columnsText = yield* execClickHouseWithClient(client, config, columnsSql) const columnRows = parseJsonEachRow(columnsText) const colsByTable = new Map>() @@ -898,6 +899,7 @@ export class OrgClickHouseSettingsService extends Context.Service< make: Effect.gen(function* () { const database = yield* Database const env = yield* Env + const httpClient = yield* HttpClient.HttpClient const encryptionKey = yield* parseEncryptionKey(Redacted.value(env.MAPLE_INGEST_KEY_ENCRYPTION_KEY)) // Optional: present only inside a Worker isolate. Used to kick off the // background schema-apply Workflow. Read optionally so non-worker/test @@ -1111,7 +1113,11 @@ export class OrgClickHouseSettingsService extends Context.Service< // host or token surfaces here rather than after the user closes the // dialog. No DDL is run — applying the schema is a separate explicit // action via the diff/apply endpoints. - yield* execClickHouse({ url, user, password: plainPassword, database: dbName }, "SELECT 1") + yield* execClickHouseWithClient( + httpClient, + { url, user, password: plainPassword, database: dbName }, + "SELECT 1", + ) const encryptedPassword = plainPassword.length > 0 ? yield* encryptToken(plainPassword, encryptionKey) : null @@ -1203,7 +1209,7 @@ export class OrgClickHouseSettingsService extends Context.Service< yield* requireAdmin(roles) const row = yield* requireActiveRow(orgId) const config = yield* loadConfigForRow(row) - const actual = yield* fetchActualSchema(config) + const actual = yield* fetchActualSchema(httpClient, config) const entries = computeSchemaDiff({ tables: yield* getDesiredTables }, actual) // Self-heal the recorded schema version. The ingest gateway only routes an @@ -1753,7 +1759,7 @@ export class OrgClickHouseSettingsService extends Context.Service< } satisfies OrgClickHouseSettingsServiceShape }), }) { - static readonly layer = Layer.effect(this, this.make) + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(FetchHttpClient.layer)) static readonly get = (orgId: OrgId, roles: ReadonlyArray) => this.use((service) => service.get(orgId, roles)) diff --git a/apps/api/src/services/warehouse/WarehouseQueryService.ts b/apps/api/src/services/warehouse/WarehouseQueryService.ts index c175f3c92..a383a29a7 100644 --- a/apps/api/src/services/warehouse/WarehouseQueryService.ts +++ b/apps/api/src/services/warehouse/WarehouseQueryService.ts @@ -500,12 +500,13 @@ export class WarehouseQueryService extends Context.Service< } /** - * Layer that provides the package-level `WarehouseExecutor` for a tenant, - * backed by `WarehouseQueryService`. The executor name is a public contract - * from `@maple/query-engine`; only the wiring lives here. + * Provides the package-level `WarehouseExecutor` for a tenant from the + * request's existing `WarehouseQueryService`. The executor is a pure facade, + * so installing the service directly avoids constructing a request-local + * Layer (and the extra scope that comes with it). */ -export const makeWarehouseExecutorFromTenant = (tenant: TenantContext) => - Layer.effect( +export const provideWarehouseExecutorFromTenant = (tenant: TenantContext) => + Effect.provideServiceEffect( WarehouseExecutor, Effect.map(WarehouseQueryService, (warehouse) => warehouse.asExecutor(tenant)), ) diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index 41c2f6151..d64845ea6 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -325,6 +325,8 @@ const handle = async ( method: request.method, url: request.url, }), + // One-shot recovery fiber after the main handler runtime rejected. + // oxlint-disable-next-line effecttsgo/strict-effect-provide Effect.provide(telemetry.layer), ), ) diff --git a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts index 523dbb27c..c7f493538 100644 --- a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts +++ b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts @@ -63,7 +63,13 @@ const bustRuntimeConfigCache = (orgId: OrgId): Promise => invalidateOrgRuntimeConfigMemo(orgId) const cache = yield* EdgeCacheService yield* cache.invalidate({ bucket: ORG_CH_CONFIG_CACHE_BUCKET, key: orgId }) - }).pipe(Effect.provide(EdgeCacheService.layer.pipe(Layer.provide(CacheBackendLive))), Effect.ignore), + }).pipe( + // Promise bridge for the standalone workflow isolate; no application + // runtime exists here to own this cache layer. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + Effect.provide(EdgeCacheService.layer.pipe(Layer.provide(CacheBackendLive))), + Effect.ignore, + ), ) /** @@ -391,7 +397,14 @@ async function runWithDb( ): Promise { const orgId = Schema.decodeUnknownSync(OrgId)(event.payload.orgId) const dbStep: DbStep = (fn) => - Effect.runPromise(connection.run(fn).pipe(Effect.provide(schemaApplyTelemetry.layer))) + Effect.runPromise( + connection.run(fn).pipe( + // Each durable workflow step crosses back into Effect from Promise-land + // and owns the telemetry context for that isolated run. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + Effect.provide(schemaApplyTelemetry.layer), + ), + ) const encryptionKey = Buffer.from(env.MAPLE_INGEST_KEY_ENCRYPTION_KEY.trim(), "base64") const startedAt = Date.now() const appliedVersions: number[] = [] diff --git a/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts b/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts index a5669874a..17d91d533 100644 --- a/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts +++ b/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts @@ -606,7 +606,14 @@ async function runWithDb( const idTyped = decodeInvestigationId(investigationId) const dbStep = (fn: (db: MaplePgClient) => Promise): Promise => - Effect.runPromise(connection.run(fn).pipe(Effect.provide(fanoutTelemetry.layer))) + Effect.runPromise( + connection.run(fn).pipe( + // Each durable workflow step crosses back into Effect from Promise-land + // and owns the telemetry context for that isolated run. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + Effect.provide(fanoutTelemetry.layer), + ), + ) const laneRows = () => dbStep((db) => diff --git a/apps/api/src/workflows/agent-pass.test.ts b/apps/api/src/workflows/agent-pass.test.ts index ca1e5f6be..eda258fa5 100644 --- a/apps/api/src/workflows/agent-pass.test.ts +++ b/apps/api/src/workflows/agent-pass.test.ts @@ -91,7 +91,7 @@ const pass = ( submitToolDescription: "Record the candidate.", schema: SCHEMA, ...(options.deadlineAtMs === undefined ? {} : { deadlineAtMs: options.deadlineAtMs }), - }).pipe(Effect.provide(stub(steps)), Effect.provide(ToolExecutorStubLayer)) + }).pipe(Effect.provide(Layer.mergeAll(stub(steps), ToolExecutorStubLayer))) /** Every step calls a tool, so the agent never voluntarily stops. */ const grinding = (count: number): ReadonlyArray> => diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 5b2ddeefc..aa702a8e6 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -168,6 +168,8 @@ export type MigrationResolutionErrorKind = | "missing-path" | "chdb-mismatch" +// Migration planning is synchronous throw/catch code; this error does not enter an Effect failure channel. +// oxlint-disable-next-line effecttsgo/extends-native-error export class MigrationResolutionError extends Error { readonly kind: MigrationResolutionErrorKind constructor(kind: MigrationResolutionErrorKind, message: string) { diff --git a/apps/cli/src/server/otlp/encode.ts b/apps/cli/src/server/otlp/encode.ts index 2aad0d69a..32cd105a0 100644 --- a/apps/cli/src/server/otlp/encode.ts +++ b/apps/cli/src/server/otlp/encode.ts @@ -96,6 +96,8 @@ function base64ToBytes(b64: string): Uint8Array { * 400 from the ingest handler rather than the generic 500 an encoder crash * would produce. */ +// The pure encoder throws this sentinel for its HTTP adapter to translate into a 400 response. +// oxlint-disable-next-line effecttsgo/extends-native-error export class OtlpFieldError extends Error { constructor(message: string) { super(message) diff --git a/apps/electric-sync/src/worker.ts b/apps/electric-sync/src/worker.ts index f60e94ee1..d48607a49 100644 --- a/apps/electric-sync/src/worker.ts +++ b/apps/electric-sync/src/worker.ts @@ -138,6 +138,8 @@ const handle = async ( Effect.runFork( Effect.logError("electric-sync handler failed").pipe( Effect.annotateLogs({ error: message }), + // One-shot recovery fiber after the main handler runtime rejected. + // oxlint-disable-next-line effecttsgo/strict-effect-provide Effect.provide(telemetry.layer), ), ) diff --git a/apps/web/src/api/warehouse/custom-charts.test.ts b/apps/web/src/api/warehouse/custom-charts.test.ts index 0a5a63a9e..2ec711a1d 100644 --- a/apps/web/src/api/warehouse/custom-charts.test.ts +++ b/apps/web/src/api/warehouse/custom-charts.test.ts @@ -8,7 +8,7 @@ const executeQueryEngineMock = vi.fn() vi.mock("@/api/warehouse/effect-utils", () => ({ WarehouseDateTimeString: Schema.String, decodeInput: (_schema: unknown, data: unknown) => Effect.succeed(data), - invalidWarehouseInput: () => Effect.fail(new Error("invalid")), + invalidWarehouseInput: () => Effect.fail("invalid"), executeQueryEngine: (...args: unknown[]) => executeQueryEngineMock(...args), })) diff --git a/apps/web/src/api/warehouse/services.test.ts b/apps/web/src/api/warehouse/services.test.ts index e84db0cef..39734d528 100644 --- a/apps/web/src/api/warehouse/services.test.ts +++ b/apps/web/src/api/warehouse/services.test.ts @@ -10,7 +10,7 @@ const runWarehouseQueryMock = vi.fn() vi.mock("@/api/warehouse/effect-utils", () => ({ WarehouseDateTimeString: Schema.String, decodeInput: (_schema: unknown, data: unknown) => Effect.succeed(data), - invalidWarehouseInput: () => Effect.fail(new Error("invalid")), + invalidWarehouseInput: () => Effect.fail("invalid"), extractFacets: () => [], executeQueryEngine: (...args: unknown[]) => executeQueryEngineMock(...args), runWarehouseQuery: (_operation: string, execute: () => unknown) => diff --git a/apps/web/src/api/warehouse/timeseries-adapters.test.ts b/apps/web/src/api/warehouse/timeseries-adapters.test.ts index da0ee423d..41ff68cb5 100644 --- a/apps/web/src/api/warehouse/timeseries-adapters.test.ts +++ b/apps/web/src/api/warehouse/timeseries-adapters.test.ts @@ -13,7 +13,7 @@ vi.mock("@/api/warehouse/effect-utils", () => ({ _tag = "WarehouseQueryError" }, decodeInput: (_schema: unknown, data: unknown) => Effect.succeed(data), - invalidWarehouseInput: () => Effect.fail(new Error("invalid")), + invalidWarehouseInput: () => Effect.fail("invalid"), executeQueryEngine: (...args: unknown[]) => executeQueryEngineMock(...args), runWarehouseQuery: (...args: unknown[]) => runWarehouseQueryMock(...args), })) diff --git a/apps/web/src/lib/models/alerts-overview-model.registry.test.ts b/apps/web/src/lib/models/alerts-overview-model.registry.test.ts index 44c20f23e..cd9736456 100644 --- a/apps/web/src/lib/models/alerts-overview-model.registry.test.ts +++ b/apps/web/src/lib/models/alerts-overview-model.registry.test.ts @@ -77,6 +77,11 @@ interface UpdateRuleReq { readonly payload: { readonly enabled?: boolean } } +class TestUpdateRuleError extends Schema.TaggedError()( + "@maple/web/test/TestUpdateRuleError", + { message: Schema.String }, +) {} + /** * A fake MapleApiV2AtomClient exposing only `alertRules.update` (all the * handler touches), recording every call. The real client is a large generated @@ -137,7 +142,7 @@ describe("AlertsOverviewModel toggle mutation", () => { }) it.effect("records a failed toggle in `state` and surfaces the typed error", () => { - const fake = makeFakeClient(() => Effect.fail(new Error("nope"))) + const fake = makeFakeClient(() => Effect.fail(new TestUpdateRuleError({ message: "nope" }))) return Effect.gen(function* () { const ports = yield* Model.get(ToggleTestModel) const exit = yield* Mutation.call(ports.inputs.toggle, makeRule()).pipe(Effect.exit) diff --git a/examples/alchemy-maple/src/Api.ts b/examples/alchemy-maple/src/Api.ts index 558bf21a5..bb90e20c3 100644 --- a/examples/alchemy-maple/src/Api.ts +++ b/examples/alchemy-maple/src/Api.ts @@ -83,6 +83,7 @@ export default class Api extends Cloudflare.Worker()( // handler's requirements — and it is cheap: the layer is a few // references over buffers that live in `telemetry`, so the per-event // build keeps nothing of its own. + // oxlint-disable-next-line effecttsgo/strict-effect-provide Effect.provide(telemetry.layer), ), } diff --git a/examples/effect-todo/server/TodoService.ts b/examples/effect-todo/server/TodoService.ts index 5b7397c78..79e0cdb9f 100644 --- a/examples/effect-todo/server/TodoService.ts +++ b/examples/effect-todo/server/TodoService.ts @@ -10,7 +10,7 @@ * - `toggle` fails ~15% of the time with `ToggleFailedError` so Maple's Errors * view and error-rate metrics have data. */ -import { Context, Duration, Effect, Layer, Ref } from "effect" +import { Context, Duration, Effect, Layer, Random, Ref } from "effect" import { Todo, TodoNotFoundError, ToggleFailedError } from "../shared/api.ts" const seedTodos: ReadonlyArray = [ @@ -36,7 +36,9 @@ const seedTodos: ReadonlyArray = [ /** Sleep a random number of ms in [min, max] to make spans visibly wide. */ const jitter = (minMs: number, maxMs: number) => - Effect.suspend(() => Effect.sleep(Duration.millis(minMs + Math.floor(Math.random() * (maxMs - minMs))))) + Random.nextBetween(minMs, maxMs).pipe( + Effect.flatMap((delayMs) => Effect.sleep(Duration.millis(Math.floor(delayMs)))), + ) export class TodoService extends Context.Service()("@maple-examples/todo/TodoService", { make: Effect.gen(function* () { @@ -82,7 +84,7 @@ export class TodoService extends Context.Service()("@maple-examples // The simulated flake: a slow write that occasionally loses a race. yield* jitter(40, 160) - if (Math.random() < 0.15) { + if ((yield* Random.next) < 0.15) { yield* Effect.logWarning("todo.toggle.conflict").pipe(Effect.annotateLogs({ "todo.id": id })) return yield* new ToggleFailedError({ message: `Transient write conflict toggling ${id}` }) } diff --git a/packages/alchemy-maple/test/providers.test.ts b/packages/alchemy-maple/test/providers.test.ts index 1a3de9aeb..af17b56db 100644 --- a/packages/alchemy-maple/test/providers.test.ts +++ b/packages/alchemy-maple/test/providers.test.ts @@ -52,11 +52,17 @@ const wireDashboard = { updated_at: "2026-07-01T12:00:00.000Z", } -const runWithProvider = (api: MapleApiShape, program: Effect.Effect) => - program.pipe( - Effect.provide(DashboardProvider().pipe(Layer.provide(Layer.succeed(MapleApi, api)))), - Effect.provide(ApiKeyProvider().pipe(Layer.provide(Layer.succeed(MapleApi, api)))), +const runWithProvider = (api: MapleApiShape, program: Effect.Effect) => { + const apiLayer = Layer.succeed(MapleApi, api) + return program.pipe( + Effect.provide( + Layer.mergeAll( + DashboardProvider().pipe(Layer.provide(apiLayer)), + ApiKeyProvider().pipe(Layer.provide(apiLayer)), + ), + ), ) +} describe("DashboardProvider", () => { it.live("creates when there is no prior state", () => diff --git a/packages/clickhouse-cli/src/client.ts b/packages/clickhouse-cli/src/client.ts index 7ba92131b..6d39ca5b8 100644 --- a/packages/clickhouse-cli/src/client.ts +++ b/packages/clickhouse-cli/src/client.ts @@ -45,6 +45,8 @@ export async function exec(config: ClickHouseConfig, sql: string): Promise(), /** Per-connection HMAC secret for inbound PlanetScale webhooks. */ webhookSecretCiphertext: text("webhook_secret_ciphertext"), webhookSecretIv: text("webhook_secret_iv"), diff --git a/packages/db/src/schema/scrape-targets.ts b/packages/db/src/schema/scrape-targets.ts index 6dd43e84b..83fa3e3d9 100644 --- a/packages/db/src/schema/scrape-targets.ts +++ b/packages/db/src/schema/scrape-targets.ts @@ -1,10 +1,10 @@ -import type { OrgId } from "@maple/domain" +import type { OrgId, ScrapeTargetId } from "@maple/domain" import { boolean, index, integer, jsonb, pgTable, text, timestamp } from "drizzle-orm/pg-core" export const scrapeTargets = pgTable( "scrape_targets", { - id: text("id").notNull().primaryKey(), + id: text("id").$type().notNull().primaryKey(), orgId: text("org_id").$type().notNull(), name: text("name").notNull(), serviceName: text("service_name"), @@ -51,6 +51,7 @@ export const scrapeTargetChecks = pgTable( // carry over existing ids; setval() realigns the sequence afterwards. id: integer("id").primaryKey().generatedByDefaultAsIdentity(), targetId: text("target_id") + .$type() .notNull() .references(() => scrapeTargets.id, { onDelete: "cascade" }), orgId: text("org_id").$type().notNull(), diff --git a/packages/query-engine/src/caching/bucket-cache.hit-rate.test.ts b/packages/query-engine/src/caching/bucket-cache.hit-rate.test.ts index f9419acf8..4aa9b491c 100644 --- a/packages/query-engine/src/caching/bucket-cache.hit-rate.test.ts +++ b/packages/query-engine/src/caching/bucket-cache.hit-rate.test.ts @@ -122,6 +122,9 @@ const makeLive = (backend: EdgeCacheBackend, readTimeoutMs?: number) => Layer.provide(Layer.succeed(EdgeCacheService, makeEdgeCacheService(backend, readTimeoutMs))), ) +const makeTestLive = (backend: EdgeCacheBackend, config = makeConfig(), readTimeoutMs?: number) => + makeLive(backend, readTimeoutMs).pipe(Layer.provide(config)) + interface Request { readonly startMs: number readonly endMs: number @@ -160,7 +163,7 @@ const runSequence = (requests: ReadonlyArray, backend: EdgeCacheBackend } return { outcomes, warehouseCalls } - }).pipe(Effect.provide(makeLive(backend)), Effect.provide(config)) + }).pipe(Effect.provide(makeTestLive(backend, config))) /** The same sequence with the cache bypassed — ground truth for the points. */ const oracle = (requests: ReadonlyArray) => @@ -338,7 +341,7 @@ describe("bucket cache hit rate — dashboard access patterns", () => { for (const point of b.points) { assert.strictEqual(point.series.owner, 2) } - }).pipe(Effect.provide(makeLive(backend)), Effect.provide(makeConfig())) + }).pipe(Effect.provide(makeTestLive(backend))) }) it.live("refills from the warehouse after the entries expire", () => { @@ -444,10 +447,7 @@ describe("bucket cache hit rate — degraded backends", () => { { orgId, query: DEFAULT_QUERY, bucketSeconds: BUCKET_SECONDS, ...fixed }, compute, ) - }).pipe( - Effect.provide(makeLive(hangingReads(inner), READ_TIMEOUT_MS)), - Effect.provide(makeConfig()), - ) + }).pipe(Effect.provide(makeTestLive(hangingReads(inner), makeConfig(), READ_TIMEOUT_MS))) assert.strictEqual(degraded.segmentsTimedOut, 1) assert.deepStrictEqual(degraded.points, warehousePointsFor(fixed.startMs, fixed.endMs)) @@ -461,7 +461,7 @@ describe("bucket cache hit rate — degraded backends", () => { { orgId, query: DEFAULT_QUERY, bucketSeconds: BUCKET_SECONDS, ...fixed }, compute, ) - }).pipe(Effect.provide(makeLive(inner)), Effect.provide(makeConfig())) + }).pipe(Effect.provide(makeTestLive(inner))) assert.strictEqual(recovered.bucketsMissed, 0) assert.strictEqual(recovered.warehouseQueryCount, 0) @@ -493,7 +493,7 @@ describe("bucket cache hit rate — degraded backends", () => { assert.deepStrictEqual(outcome.points, warehousePointsFor(fixed.startMs, fixed.endMs)) assert.strictEqual(outcome.segmentsErrored, 1) - }).pipe(Effect.provide(makeLive(exploding)), Effect.provide(makeConfig())) + }).pipe(Effect.provide(makeTestLive(exploding))) }) it.live("round-trips a segment payload through JSON without losing buckets", () => { @@ -576,6 +576,6 @@ describe("bucket cache hit rate — concurrency", () => { `widget ${index} received points from a different window`, ) }) - }).pipe(Effect.provide(makeLive(backend)), Effect.provide(makeConfig())) + }).pipe(Effect.provide(makeTestLive(backend))) }) }) diff --git a/packages/query-engine/src/caching/bucket-cache.test.ts b/packages/query-engine/src/caching/bucket-cache.test.ts index 342de32e2..e5f45e859 100644 --- a/packages/query-engine/src/caching/bucket-cache.test.ts +++ b/packages/query-engine/src/caching/bucket-cache.test.ts @@ -326,6 +326,11 @@ const makeBucketLive = (backend: EdgeCacheBackend, readTimeoutMs?: number) => Layer.provide(Layer.succeed(EdgeCacheService, makeEdgeCacheService(backend, readTimeoutMs))), ) +const BucketTestLive = BucketLive.pipe(Layer.provide(makeConfig())) + +const makeBucketTestLive = (backend: EdgeCacheBackend, config = makeConfig(), readTimeoutMs?: number) => + makeBucketLive(backend, readTimeoutMs).pipe(Layer.provide(config)) + // These exercise the live cache backend and compute flux boundaries relative to // the real clock, so they run under it.live rather than the default TestClock. describe("BucketCacheService.getOrComputeBuckets", () => { @@ -365,7 +370,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(second.missingRangeCount, 0) assert.strictEqual(second.warehouseQueryCount, 0) assert.strictEqual(second.points.length, 3) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) it.live("refetches only the tail slice when the window shifts forward", () => { @@ -400,7 +405,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.deepStrictEqual(computeCalls[1], { startMs: 3 * MIN, endMs: 4 * MIN }) assert.strictEqual(second.missingRangeCount, 1) assert.strictEqual(second.points.length, 3) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) it.live("issues one warehouse query when the cached buckets are fragmented", () => { @@ -441,7 +446,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { // Over-fetching the cached middle must not corrupt the result set. assert.strictEqual(outcome.points.length, 6) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) it.live("propagates errors from compute and does not poison the cache", () => { @@ -479,7 +484,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(computeAttempt, 2) // first failed, second recomputed (no poison) assert.strictEqual(ok.points.length, 3) assert.isTrue(ok.bucketsMissed > 0) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) it.live("bypasses cache on a bucketSeconds mismatch (different fingerprint)", () => { @@ -512,7 +517,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(computeCalls[0]!.bucketSeconds, 60) assert.strictEqual(computeCalls[1]!.bucketSeconds, 180) assert.isTrue(second.bucketsMissed > 0) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) it.live("treats a version-skewed cache payload as a miss", () => { @@ -562,7 +567,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(computed, 1) assert.isTrue(outcome.bucketsMissed > 0) assert.strictEqual(outcome.points.length, 3) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) it.live("treats a malformed segment payload as a miss instead of defecting", () => { @@ -591,7 +596,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(computed, 1) assert.strictEqual(outcome.segmentsMissed, 1) assert.strictEqual(outcome.points.length, 1) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) it.live("stores bounded fixed-size segments instead of one growing query blob", () => { @@ -636,8 +641,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(segment.segmentEndMs - segment.segmentStartMs, 2 * MIN) } }).pipe( - Effect.provide(makeBucketLive(backend)), - Effect.provide(makeConfig({ QE_BUCKET_CACHE_SEGMENT_BUCKETS: "2" })), + Effect.provide(makeBucketTestLive(backend, makeConfig({ QE_BUCKET_CACHE_SEGMENT_BUCKETS: "2" }))), ) }) @@ -681,11 +685,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { writes[0]!.buckets.map((b) => b.startMs), [0, MIN], ) - }).pipe( - Effect.provide(makeBucketLive(backend, 10)), - Effect.provide(makeConfig()), - Effect.timeout(200), - ) + }).pipe(Effect.provide(makeBucketTestLive(backend, makeConfig(), 10)), Effect.timeout(200)) }) it.live("caches explicit empty-bucket coverage for sparse query results", () => { @@ -712,7 +712,7 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(second.bucketsHit, 3) assert.strictEqual(second.warehouseQueryCount, 0) assert.deepStrictEqual(second.points, []) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) // Uses a real wall-clock sleep to keep both fibers in-flight simultaneously, @@ -746,6 +746,6 @@ describe("BucketCacheService.getOrComputeBuckets", () => { assert.strictEqual(computeCalls, 2) assert.strictEqual(a.points.length, 3) assert.strictEqual(b.points.length, 3) - }).pipe(Effect.provide(BucketLive), Effect.provide(makeConfig())) + }).pipe(Effect.provide(BucketTestLive)) }) }) From 8f582de50a10e51172545ecb3f047bfef790bc33 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 12 Aug 2026 00:55:33 +0200 Subject: [PATCH 2/2] feat(services): derive the Apdex threshold from a detected app kind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apdex on the service detail pane was scored against a fixed T = 500 ms for every service. That is a target for a backend API and meaningless for a browser app, where an entry span is a request from a device on someone's home wifi — such a service reads as permanently frustrated and the chart stops carrying signal. Detection. `service_platforms_hourly` already answered "where does this service run"; it could not answer "what kind of app is this", because the only signal it carried for that was `maple.sdk.type`, present solely on services using a Maple SDK. Migration 0015 adds the vendor-neutral markers (`telemetry.sdk.language`, `browser.platform`, `device.type`) so a customer on vanilla OTel browser JS is classified too. `classifyServiceAppKind` resolves browser | mobile | backend | unknown, checking browser first: a browser app can carry `cloud.provider` from a CDN or a `k8s.*` leak from a gateway it was proxied through, while a server never reports `browser.platform`. Two things this surfaced that were already broken: - Maple's own browser SDK writes `maple.sdk.type = "browser"` but the platform classifier only matched `"client"`, so every browser service was classifying as `unknown`. - The service detail page never fetched platform data at all; the standalone `servicePlatforms` handler had no caller. Threshold. browser 2500 ms (the Core Web Vitals "good" LCP boundary, which puts the frustrated line at 4T = 10 s), mobile 1000 ms, everything else unchanged at 500 ms. `canUseAnnualServiceOverview` requires `apdexThresholdMs === 500`, and the Overview chart is a single `allMetrics` request — threading a browser target through it would drop throughput, latency AND error rate onto the 30-day raw path for the sake of one series. So `payload.timeseries` is forwarded untouched and Apdex is re-scored by a second, narrower query, only when T differs from the default. That override reads `service_overview_spans` (30-day TTL) while the rest of the chart reaches a year back, so on a longer range its early buckets have no score. `apdexScore` widens to `number | null` and those buckets render as gaps: carrying the 500 ms number through would silently mix two thresholds in one series, and zero would draw a crater reading as "every user was frustrated". Migration 0015 is `requiredForIngest: false` — `service_platforms_hourly` is filled by a materialized view, never by a native INSERT, so a BYO cluster that has not applied it keeps ingesting correctly and simply classifies its services exactly as it does today. There is no backfill: `max()` over the viewed window means one hour of fresh telemetry classifies a service, and the table's one `sum` column would double-count if re-inserted. --- apps/api/src/routes/v1/query-engine.http.ts | 77 +- apps/cli/src/server/local-schema-history.ts | 7 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 2 + .../v4-to-v5-service-app-kind.ts | 246 +++ apps/cli/src/server/schema-identity.ts | 17 +- apps/cli/src/server/schema/local-inserts.json | 2 +- .../cli/src/server/schema/local-schema-v5.sql | 1702 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 10 +- apps/cli/test/local-store-migrations.test.ts | 34 +- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- .../src/api/warehouse/custom-charts.test.ts | 46 + apps/web/src/api/warehouse/custom-charts.ts | 42 +- apps/web/src/api/warehouse/services.ts | 8 +- .../services/service-app-kind-badge.tsx | 74 + apps/web/src/routes/services/$serviceName.tsx | 36 +- packages/domain/package.json | 1 + .../migrations/0015_service_app_kind.ts | 74 + .../src/clickhouse/migrations/index.test.ts | 42 +- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 6 +- .../generated/tinybird-project-manifest.ts | 6 +- packages/domain/src/http/query-engine.ts | 26 + packages/domain/src/service-app-kind.test.ts | 96 + packages/domain/src/service-app-kind.ts | 115 ++ packages/domain/src/tinybird/datasources.ts | 12 +- .../domain/src/tinybird/materializations.ts | 9 +- packages/query-engine/src/ch/ch.test.ts | 31 +- .../src/ch/queries/service-map.test.ts | 23 + .../src/ch/queries/service-map.ts | 15 + packages/query-engine/src/ch/tables.ts | 3 + packages/query-engine/src/registry/queries.ts | 23 + scripts/check-local-schema-manifest.ts | 15 + 33 files changed, 2770 insertions(+), 36 deletions(-) create mode 100644 apps/cli/src/server/local-store-migrations/v4-to-v5-service-app-kind.ts create mode 100644 apps/cli/src/server/schema/local-schema-v5.sql create mode 100644 apps/web/src/components/services/service-app-kind-badge.tsx create mode 100644 packages/domain/src/clickhouse/migrations/0015_service_app_kind.ts create mode 100644 packages/domain/src/service-app-kind.test.ts create mode 100644 packages/domain/src/service-app-kind.ts diff --git a/apps/api/src/routes/v1/query-engine.http.ts b/apps/api/src/routes/v1/query-engine.http.ts index dceeab780..ecc70e61e 100644 --- a/apps/api/src/routes/v1/query-engine.http.ts +++ b/apps/api/src/routes/v1/query-engine.http.ts @@ -71,6 +71,12 @@ import { TraceId, SpanId, } from "@maple/domain/http" +import { + apdexThresholdMsForAppKind, + classifyServiceAppKind, + DEFAULT_APDEX_THRESHOLD_MS, + type ServiceAppKind, +} from "@maple/domain/service-app-kind" import { Clock, Effect, Match, Option, Schema } from "effect" import { QueryEngineService } from "@/services/warehouse/QueryEngineService" import { makeDirectRouteCachePolicy, makeExecuteRawSql } from "@maple/query-engine/runtime" @@ -233,9 +239,16 @@ const toServicePlatformRow = (row: CH.ServicePlatformsOutput) => { const faasName = String(row.faasName ?? "") const mapleSdkType = String(row.mapleSdkType ?? "") const processRuntimeName = String(row.processRuntimeName ?? "") + // App-kind signals. Optional on the row so a query compiled before migration + // 0015 (or a cluster that has not applied it) decodes as "no signal". + const telemetrySdkLanguage = String(row.telemetrySdkLanguage ?? "") + const browserPlatform = String(row.browserPlatform ?? "") + const deviceType = String(row.deviceType ?? "") // cluster.name alone does not prove the service runs in Kubernetes. const isKubernetes = k8sPodName !== "" || k8sDeploymentName !== "" - // Host infrastructure takes precedence over SDK self-report. + // Host infrastructure takes precedence over SDK self-report. `browser` is + // what Maple's own browser SDK reports (packages/browser); `client` is the + // Effect client SDK. const platform: "kubernetes" | "cloudflare" | "lambda" | "web" | "unknown" = cloudPlatform === "cloudflare.workers" || cloudProvider === "cloudflare" ? "cloudflare" @@ -243,12 +256,23 @@ const toServicePlatformRow = (row: CH.ServicePlatformsOutput) => { ? "lambda" : isKubernetes ? "kubernetes" - : mapleSdkType === "client" + : mapleSdkType === "client" || mapleSdkType === "browser" ? "web" : "unknown" return { serviceName: decodeServiceName(String(row.serviceName ?? "")), platform, + appKind: classifyServiceAppKind({ + browserPlatform, + telemetrySdkLanguage, + mapleSdkType, + deviceType, + cloudPlatform, + cloudProvider, + faasName, + k8sPodName, + k8sDeploymentName, + }), k8sCluster, cloudPlatform, cloudProvider, @@ -922,7 +946,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", // execute-path cache; releases is uncached (mirrors the standalone // handler); environments is edge-cached on a service-scoped key. yield* warehouse.warmRoute(tenant) - const [timeseries, releaseRows, environmentRows] = yield* Effect.all( + const [timeseries, releaseRows, environmentRows, appKindRows] = yield* Effect.all( [ queryEngine.execute(tenant, payload.timeseries), runQuery(Queries.serviceReleases, tenant, payload), @@ -931,9 +955,51 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", startTime: payload.startTime, endTime: payload.endTime, }), + // What kind of app this is, which is what picks the Apdex + // target below. Runs alongside the rest — it gates only the + // optional override query, not the primary chart. + runQuery(Queries.serviceAppKind, tenant, { + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + }), ], - { concurrency: 3 }, + { concurrency: 4 }, ) + + const appKind: ServiceAppKind = + appKindRows.length > 0 ? toServicePlatformRow(appKindRows[0]!).appKind : "unknown" + const apdexThresholdMs = apdexThresholdMsForAppKind(appKind) + + // `payload.timeseries` is forwarded untouched so it keeps the + // annual service-overview rollup, whose stored Apdex counters are + // baked at 500 ms (`canUseAnnualServiceOverview` enforces that). + // Threading a different threshold through it would knock + // throughput, latency, AND error rate onto the 30-day raw path for + // the sake of one series — so a non-default target is re-scored by + // this second, narrower query instead. + const apdexOverride = + apdexThresholdMs === DEFAULT_APDEX_THRESHOLD_MS + ? undefined + : yield* runQuery(Queries.serviceApdex, tenant, { + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + bucketSeconds: + payload.timeseries.query.kind === "timeseries" + ? payload.timeseries.query.bucketSeconds + : payload.releasesBucketSeconds, + apdexThresholdMs, + }).pipe( + Effect.map((rows) => + rows.map((row) => ({ + bucket: String(row.bucket), + apdexScore: Number(row.apdexScore), + totalCount: Number(row.totalCount), + })), + ), + ) + return new ServiceDetailOverviewResponse({ timeseries, releases: releaseRows.map((row) => ({ @@ -945,6 +1011,9 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", environments: environmentRows .map((row) => String(row.environment ?? "")) .filter((env) => env !== ""), + appKind, + apdexThresholdMs, + ...(apdexOverride === undefined ? {} : { apdexOverride }), }) }), ) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index c5d051f39..60cc6ba81 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -51,4 +51,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "826f9363db5dd7722debd0c87a5b74a5b66387f4752abc219d4cc0ce76358a9e", projectRevision: "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a", }), + Object.freeze({ + version: 5, + fingerprint: "3099929d42b2ce8b", + digest: "3099929d42b2ce8b18c06a428a8e32a51ce9724300138241110e08a3f09e8193", + manifestDigest: "99d834ae3baab1d0a753f18a96b96a0130bb0af8964a0b119f1f4203e3bc6d0f", + projectRevision: "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 19b611ed5..40530abf3 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 4 as const +export const LOCAL_SCHEMA_VERSION = 5 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index aa702a8e6..16d3867e6 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -36,6 +36,7 @@ import { legacyToCurrentModule } from "./local-store-migrations/legacy-to-curren import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error-rollup" import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" +import { v4ToV5ServiceAppKindModule } from "./local-store-migrations/v4-to-v5-service-app-kind" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -119,6 +120,7 @@ export const localStoreMigrations: ReadonlyArray = v1ToV2ErrorRollupModule, v2ToV3ServiceMapIngestBridgeModule, v3ToV4WebEventsModule, + v4ToV5ServiceAppKindModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-app-kind.ts b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-app-kind.ts new file mode 100644 index 000000000..3f6f04290 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-app-kind.ts @@ -0,0 +1,246 @@ +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V4, + LOCAL_SCHEMA_V4_MANIFEST, + LOCAL_SCHEMA_V4_SQL, + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V5_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +interface V4ToV5State { + readonly module: "local-0004-to-0005-service-app-kind" + readonly version: 1 + readonly rawRows: Readonly> + readonly retentionDays?: number +} + +interface V4ToV5Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v4 -> v5 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (typeof count !== "string" || !/^\d+$/.test(count)) + throw new Error(`v4 -> v5 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v4 -> v5 rawRows contains an unknown table") + return counts +} + +const decodeState = (value: unknown): V4ToV5State => { + if (!isRecord(value)) throw new Error("v4 -> v5 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v4 -> v5 state contains an unknown field") + if (value.module !== "local-0004-to-0005-service-app-kind" || value.version !== 1) + throw new Error("v4 -> v5 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v4 -> v5 retentionDays must be an integer") + return { + module: "local-0004-to-0005-service-app-kind", + version: 1, + rawRows: decodeCounts(value.rawRows), + ...(value.retentionDays === undefined ? {} : { retentionDays: value.retentionDays }), + } +} + +const decodeProgress = (value: unknown): V4ToV5Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v4 -> v5 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V4_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V4_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V4_SQL, bootstrapSchema: false }, + ) + return { + module: "local-0004-to-0005-service-app-kind", + version: 1, + rawRows, + ...(retentionDays === undefined ? {} : { retentionDays }), + } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V4ToV5State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Three appended columns on `service_platforms_hourly` plus the view that fills + * them. Unlike v3 -> v4 the bootstrap pass alone is not enough: the table + * already exists, so its `CREATE TABLE IF NOT EXISTS` is a no-op and the + * columns would never appear. The ALTERs run first, against the v4 snapshot, + * and the view is dropped so the bootstrap recreates it with the widened + * SELECT. + * + * `SimpleAggregateFunction(max, String)` columns default to empty, which is + * exactly what the classifier reads as "no signal" — so historical hours keep + * classifying as they do today (`unknown` -> the 500 ms Apdex default) and + * converge as soon as one hour of fresh telemetry lands. Nothing is rewritten. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS TelemetrySdkLanguage SimpleAggregateFunction(max, String) AFTER ProcessRuntimeName", + ) + db.exec( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS BrowserPlatform SimpleAggregateFunction(max, String) AFTER TelemetrySdkLanguage", + ) + db.exec( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS DeviceType SimpleAggregateFunction(max, String) AFTER BrowserPlatform", + ) + db.exec("DROP VIEW IF EXISTS service_platforms_hourly_mv") + }, + { schemaSql: LOCAL_SCHEMA_V4_SQL, bootstrapSchema: false }, + ) + return context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V5_SQL, + bootstrapSchema: true, + }) +} + +const verify = async ( + context: MigrationModuleContext, + state: V4ToV5State, + _progress: V4ToV5Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V5_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v4 -> v5 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v4-store", + description: "Clone the stopped v4 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "install-app-kind-columns", + description: + "Append the app-kind signal columns to service_platforms_hourly and recreate its materialized view", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v5-schema", + description: "Verify the v5 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v4 store is cloned byte-for-byte before additive DDL runs.", + }, + { + // Appended columns only — no existing column is read, rewritten, or + // reordered, and the pre-existing rows keep every value they had. The new + // columns read as empty for historical hours, which the classifier already + // treats as "no signal" and resolves to the same 500 ms Apdex default those + // hours get today. + name: "service_platforms_hourly", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Existing rows and columns are untouched; the three appended signal columns fill forward from traces writes and are complete for any window containing one hour of post-migration telemetry.", + preservationInterval: "service_platforms_hourly retention horizon", + sourceRetentionDays: 365, + targetRetentionDays: 365, + }, +] + +export const v4ToV5ServiceAppKindModule: LocalStoreMigrationModule = { + id: "local-0004-to-0005-service-app-kind", + moduleVersion: 1, + description: + "Append telemetry.sdk.language / browser.platform / device.type app-kind signals to service_platforms_hourly", + from: LOCAL_SCHEMA_V4, + to: LOCAL_SCHEMA_V5, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index a3ba0c12a..c7105cf2c 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -3,6 +3,7 @@ import schemaV1Sql from "./schema/local-schema-v1.sql" with { type: "text" } import schemaV2Sql from "./schema/local-schema-v2.sql" with { type: "text" } import schemaV3Sql from "./schema/local-schema-v3.sql" with { type: "text" } import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } +import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -26,7 +27,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" + "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -57,6 +58,11 @@ export const LOCAL_SCHEMA_V3_MANIFEST_DIGEST = LOCAL_SCHEMA_V3_MANIFEST.digest export const LOCAL_SCHEMA_V4_SQL = schemaV4Sql export const LOCAL_SCHEMA_V4_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV4Sql) export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest +/** Immutable v5 DDL/manifest snapshot used by the v4 -> v5 module after the + * generated current schema advances. */ +export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql +export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) +export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -107,6 +113,15 @@ export const LOCAL_SCHEMA_V4: LocalSchemaIdentity = Object.freeze({ projectRevision: LOCAL_SCHEMA_HISTORY[4]!.projectRevision, }) +export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_HISTORY[5]!.version, + fingerprint: LOCAL_SCHEMA_HISTORY[5]!.fingerprint, + digest: LOCAL_SCHEMA_HISTORY[5]!.digest, + manifestDigest: LOCAL_SCHEMA_HISTORY[5]!.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, +}) + export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, fingerprint: SCHEMA_FINGERPRINT, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index cebc793d3..f3cc2a508 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a", + "projectRevision": "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v5.sql b/apps/cli/src/server/schema/local-schema-v5.sql new file mode 100644 index 000000000..4f756e503 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v5.sql @@ -0,0 +1,1702 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a +-- localSchemaVersion: 5 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + TelemetrySdkLanguage SimpleAggregateFunction(max, String), + BrowserPlatform SimpleAggregateFunction(max, String), + DeviceType SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage, + max(ResourceAttributes['browser.platform']) AS BrowserPlatform, + max(ResourceAttributes['device.type']) AS DeviceType, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 59ffa044f..4f756e503 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a --- localSchemaVersion: 4 +-- projectRevision: 7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a +-- localSchemaVersion: 5 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -568,6 +568,9 @@ CREATE TABLE IF NOT EXISTS service_platforms_hourly ( FaasName SimpleAggregateFunction(max, String), MapleSdkType SimpleAggregateFunction(max, String), ProcessRuntimeName SimpleAggregateFunction(max, String), + TelemetrySdkLanguage SimpleAggregateFunction(max, String), + BrowserPlatform SimpleAggregateFunction(max, String), + DeviceType SimpleAggregateFunction(max, String), SpanCount SimpleAggregateFunction(sum, UInt64) ) ENGINE = AggregatingMergeTree @@ -1416,6 +1419,9 @@ SELECT max(ResourceAttributes['faas.name']) AS FaasName, max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage, + max(ResourceAttributes['browser.platform']) AS BrowserPlatform, + max(ResourceAttributes['device.type']) AS DeviceType, count() AS SpanCount FROM traces WHERE ServiceName != '' diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 2df59982e..3b27fa4ba 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -12,6 +12,8 @@ import { LOCAL_SCHEMA_V3, LOCAL_SCHEMA_V3_MANIFEST, LOCAL_SCHEMA_V4, + LOCAL_SCHEMA_V4_MANIFEST, + LOCAL_SCHEMA_V5, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -53,16 +55,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v4 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("75ac856927d88d56") - expect(SCHEMA_DIGEST).toBe("75ac856927d88d56518f12c68407a8f2a199d000b6eeb8576f9c97000138f5a4") + it("matches the generated v5 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("3099929d42b2ce8b") + expect(SCHEMA_DIGEST).toBe("3099929d42b2ce8b18c06a428a8e32a51ce9724300138241110e08a3f09e8193") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(4) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V4) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(5) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V5) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -85,7 +87,7 @@ describe("current local schema identity", () => { ) expect(v2TimeOrderedErrors?.definition).toContain("FROM error_events") - // v4 is exactly v3 plus the web analytics fact table and its view. Asserted + // v4 was exactly v3 plus the web analytics fact table and its view. Asserted // against the frozen v3 manifest rather than the diff so a later structural // change can't quietly ride along on this version. const webEvents = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "web_events") @@ -99,6 +101,22 @@ describe("current local schema identity", () => { expect( LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name).filter((name) => !v3Names.has(name)), ).toEqual(["web_events", "web_events_mv"]) + + // v5 adds no objects at all — only three app-kind signal columns on an + // existing table. Asserting the object sets are identical is what stops a + // structural change riding along on a column-only version. + expect(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)).toEqual( + LOCAL_SCHEMA_V4_MANIFEST.objects.map((object) => object.name), + ) + const appKindColumns = ["TelemetrySdkLanguage", "BrowserPlatform", "DeviceType"] + const platformsColumnsOf = (manifest: LocalSchemaManifest) => + manifest.objects + .find((object) => object.name === "service_platforms_hourly") + ?.columns.map((column) => column.name) ?? [] + expect(platformsColumnsOf(LOCAL_SCHEMA_MANIFEST)).toEqual(expect.arrayContaining(appKindColumns)) + expect(platformsColumnsOf(LOCAL_SCHEMA_V4_MANIFEST)).not.toEqual( + expect.arrayContaining(appKindColumns), + ) }) }) @@ -110,12 +128,14 @@ describe("local migration registry", () => { "local-0001-to-0002-error-rollup", "local-0002-to-0003-service-map-ingest-bridge", "local-0003-to-0004-web-events", + "local-0004-to-0005-service-app-kind", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) expect(chain[1]?.to).toEqual(LOCAL_SCHEMA_V2) expect(chain[2]?.to).toEqual(LOCAL_SCHEMA_V3) expect(chain[3]?.to).toEqual(LOCAL_SCHEMA_V4) + expect(chain[4]?.to).toEqual(LOCAL_SCHEMA_V5) expect(typeof chain[0]?.apply).toBe("function") }) @@ -151,7 +171,7 @@ describe("local migration registry", () => { ).toThrow(/no registered/) expect(() => resolveMigrationChain( - { ...CURRENT_LOCAL_SCHEMA, version: 5, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 6, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 08ff5ec91..a97f777f3 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a"; +pub const PROJECT_REVISION: &str = "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/web/src/api/warehouse/custom-charts.test.ts b/apps/web/src/api/warehouse/custom-charts.test.ts index 2ec711a1d..a48f3c669 100644 --- a/apps/web/src/api/warehouse/custom-charts.test.ts +++ b/apps/web/src/api/warehouse/custom-charts.test.ts @@ -16,6 +16,7 @@ import { fillServiceDetailPoints, getCustomChartServiceDetail, getServiceDetailThroughputRefinement, + mergeApdexOverride, mergeExactThroughput, } from "@/api/warehouse/custom-charts" import type { ServiceDetailTimeSeriesPoint } from "@/api/warehouse/services" @@ -182,3 +183,48 @@ describe("fillServiceDetailPoints", () => { expect(result.every((p) => p.partial === false)).toBe(true) }) }) + +describe("mergeApdexOverride", () => { + const point = (bucket: string, apdexScore: number | null): ServiceDetailTimeSeriesPoint => ({ + bucket, + throughput: 10, + tracedThroughput: 10, + hasSampling: false, + samplingWeight: 1, + errorRate: 0, + p50LatencyMs: 0, + p95LatencyMs: 0, + p99LatencyMs: 0, + apdexScore, + totalCount: 10, + partial: false, + }) + + it("replaces the 500ms-scored series with the kind-aware one", () => { + const points = [point("2026-02-01T00:00:00.000Z", 0.2), point("2026-02-01T01:00:00.000Z", 0.3)] + const merged = mergeApdexOverride( + points, + new Map([ + ["2026-02-01T00:00:00.000Z", 0.94], + ["2026-02-01T01:00:00.000Z", 0.91], + ]), + ) + + expect(merged.map((p) => p.apdexScore)).toEqual([0.94, 0.91]) + // Only apdex is re-scored — the rest of the point still comes from the + // annual rollup the primary timeseries stayed on. + expect(merged[0].throughput).toBe(10) + }) + + // The override reads `service_overview_spans` (30-day TTL) while the rest of + // the chart reaches a year back. Carrying the 500ms number through for the + // uncovered buckets would mix two thresholds in one series; 0 would draw a + // crater that reads as "every user was frustrated". + it("nulls buckets the override does not cover rather than keeping or zeroing them", () => { + const points = [point("2026-01-01T00:00:00.000Z", 0.2), point("2026-02-01T00:00:00.000Z", 0.3)] + const merged = mergeApdexOverride(points, new Map([["2026-02-01T00:00:00.000Z", 0.88]])) + + expect(merged[0].apdexScore).toBe(null) + expect(merged[1].apdexScore).toBe(0.88) + }) +}) diff --git a/apps/web/src/api/warehouse/custom-charts.ts b/apps/web/src/api/warehouse/custom-charts.ts index e8752464d..76258130d 100644 --- a/apps/web/src/api/warehouse/custom-charts.ts +++ b/apps/web/src/api/warehouse/custom-charts.ts @@ -31,6 +31,7 @@ import { invalidWarehouseInput, runWarehouseQuery, } from "@/api/warehouse/effect-utils" +import { DEFAULT_APDEX_THRESHOLD_MS, type ServiceAppKind } from "@maple/domain/service-app-kind" import { MapleApiAtomClient } from "@/lib/services/common/atom-client" import type { ServiceDetailTimeSeriesPoint, ServiceTimeSeriesPoint } from "@/api/warehouse/services" const dateTimeString = WarehouseDateTimeString @@ -837,6 +838,31 @@ export interface ServiceDetailOverviewResult { data: ServiceDetailTimeSeriesPoint[] releases: ReadonlyArray<{ bucket: string; commitSha: CommitSha; count: number; errorCount: number }> environments: string[] + /** What kind of app this service is — drives the header badge. */ + appKind: ServiceAppKind + /** The Apdex target `data[].apdexScore` was scored against, in ms. Shown on + * the chart: a score is uninterpretable without the T that produced it. */ + apdexThresholdMs: number +} + +/** + * Replace the 500 ms-scored Apdex series with one re-scored at the service's own + * target, as returned by the `serviceDetailOverview` handler. + * + * Buckets the override does not cover become `null`, not 0. The override reads + * `service_overview_spans` (30-day TTL) while the rest of the chart can reach a + * year back, so on a long range the early buckets genuinely have no score — + * carrying the 500 ms number through for those would silently mix two + * thresholds in one series, and zero would draw a crater. + */ +export function mergeApdexOverride( + points: ReadonlyArray, + overrideByBucket: ReadonlyMap, +): ServiceDetailTimeSeriesPoint[] { + return points.map((point) => ({ + ...point, + apdexScore: overrideByBucket.get(point.bucket) ?? null, + })) } export function getServiceDetailOverview({ data }: { data: GetCustomChartServiceDetailInput }) { @@ -879,8 +905,20 @@ const getServiceDetailOverviewEffect = Effect.fn("QueryEngine.getServiceDetailOv }), ) + const points = buildServiceDetailPoints(result.timeseries, startTime, endTime, bucketSeconds, nowMs) + // Present only when the service's app kind sets a target other than the + // 500 ms the primary timeseries is scored at. Absent on an older API build, + // which is the same as "the default applies". + const apdexOverride = result.apdexOverride + return { - data: buildServiceDetailPoints(result.timeseries, startTime, endTime, bucketSeconds, nowMs), + data: + apdexOverride === undefined + ? points + : mergeApdexOverride( + points, + new Map(apdexOverride.map((row) => [toIsoBucket(row.bucket), row.apdexScore])), + ), releases: result.releases.map((r) => ({ bucket: toIsoBucket(r.bucket), commitSha: r.commitSha, @@ -889,6 +927,8 @@ const getServiceDetailOverviewEffect = Effect.fn("QueryEngine.getServiceDetailOv errorCount: Number(r.errorCount ?? 0), })), environments: [...result.environments], + appKind: result.appKind ?? "unknown", + apdexThresholdMs: result.apdexThresholdMs ?? DEFAULT_APDEX_THRESHOLD_MS, } satisfies ServiceDetailOverviewResult }) diff --git a/apps/web/src/api/warehouse/services.ts b/apps/web/src/api/warehouse/services.ts index c8b006fd4..9694d3485 100644 --- a/apps/web/src/api/warehouse/services.ts +++ b/apps/web/src/api/warehouse/services.ts @@ -546,7 +546,13 @@ export interface ServiceDetailTimeSeriesPoint { p50LatencyMs: number p95LatencyMs: number p99LatencyMs: number - apdexScore: number + /** + * `null` where the score is unknown rather than zero. A service whose app + * kind sets a non-default Apdex target is re-scored from + * `service_overview_spans` (30-day TTL), so on a longer range the early + * buckets have no score — and 0 would render as "everyone was frustrated". + */ + apdexScore: number | null totalCount: number /** * The bucket is still settling — its window ends within the ingestion-lag diff --git a/apps/web/src/components/services/service-app-kind-badge.tsx b/apps/web/src/components/services/service-app-kind-badge.tsx new file mode 100644 index 000000000..ae58001e7 --- /dev/null +++ b/apps/web/src/components/services/service-app-kind-badge.tsx @@ -0,0 +1,74 @@ +import { SERVICE_APP_KIND_LABELS, type ServiceAppKind } from "@maple/domain/service-app-kind" +import { Badge } from "@maple/ui/components/ui/badge" +import { cn } from "@maple/ui/lib/utils" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { GlobeIcon, MobileIcon, ServerIcon } from "@/components/icons" +import { getServiceDetailOverviewResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" + +interface ServiceAppKindBadgeProps { + serviceName: string + startTime?: string + endTime?: string + /** Mirrors the Overview tab's bundle-atom input so this shares that fetch + * rather than issuing its own — same contract as + * `ServiceEnvironmentSwitcher`. */ + environments?: string[] + className?: string +} + +// Same token-based palette convention as `DependencyTypeBadge`: every tone maps +// onto an existing chart/severity token so the badge tracks the theme. +const tones: Record, string> = { + browser: "bg-chart-2/10 text-chart-2", + mobile: "bg-chart-4/10 text-chart-4", + backend: "bg-foreground/5 text-muted-foreground", +} + +function getIcon(kind: Exclude) { + switch (kind) { + case "browser": + return GlobeIcon + case "mobile": + return MobileIcon + case "backend": + return ServerIcon + } +} + +/** + * What kind of app this service is, derived from its resource attributes (see + * `classifyServiceAppKind`). It is not decoration: the same classification picks + * the Apdex target the Overview chart is scored against, so the badge is what + * makes that number's basis visible on the page. + * + * Renders nothing for `unknown` — a badge that says the product could not tell + * is worse than no badge, and `unknown` resolves to the same default target as + * `backend` anyway. + */ +export function ServiceAppKindBadge({ + serviceName, + startTime, + endTime, + environments, + className, +}: ServiceAppKindBadgeProps) { + const overviewResult = useAtomValue( + getServiceDetailOverviewResultAtom({ + data: { serviceName, startTime, endTime, environments }, + }), + ) + + const kind = Result.builder(overviewResult) + .onSuccess((response) => response.appKind) + .orElse((): ServiceAppKind => "unknown") + + if (kind === "unknown") return null + + const Icon = getIcon(kind) + return ( + + + {SERVICE_APP_KIND_LABELS[kind]} + + ) +} diff --git a/apps/web/src/routes/services/$serviceName.tsx b/apps/web/src/routes/services/$serviceName.tsx index 217d067d7..cc828ca23 100644 --- a/apps/web/src/routes/services/$serviceName.tsx +++ b/apps/web/src/routes/services/$serviceName.tsx @@ -15,6 +15,7 @@ import { getServiceDetailThroughputRefinementResultAtom, } from "@/lib/services/atoms/warehouse-query-atoms" import { mergeExactThroughput } from "@/api/warehouse/custom-charts" +import { DEFAULT_APDEX_THRESHOLD_MS } from "@maple/domain/service-app-kind" import type { ServiceDetailTimeSeriesPoint } from "@/api/warehouse/services" import { useCommitMarkers } from "@/components/vcs/commit-markers/use-commit-markers" import type { ReleasePoint } from "@/components/vcs/commit-markers/marker-layout" @@ -26,6 +27,7 @@ import { BellIcon } from "@/components/icons" import { ServiceDependenciesTab } from "@/components/services/service-dependencies-tab" import { ServiceOperationsTab } from "@/components/services/service-operations-tab" import { ServiceDependencyStrip } from "@/components/services/service-dependency-strip" +import { ServiceAppKindBadge } from "@/components/services/service-app-kind-badge" import { ServiceEnvironmentSwitcher } from "@/components/services/service-environment-switcher" import { ServiceErrorsPanel } from "@/components/services/service-errors-panel" import { ServiceRecentDeploys } from "@/components/services/service-recent-deploys" @@ -108,6 +110,13 @@ const SERVICE_CHARTS: ServiceChartConfig[] = [ }, ] +/** "500ms" / "2.5s" — sub-second targets stay in ms, the rest read as seconds. */ +function formatApdexTarget(thresholdMs: number): string { + if (thresholdMs < 1000) return `${thresholdMs}ms` + const seconds = thresholdMs / 1000 + return `${Number.isInteger(seconds) ? seconds : seconds.toFixed(1)}s` +} + function ServiceDetailPage() { const search = Route.useSearch() return ( @@ -194,6 +203,14 @@ function ServiceDetailContent() { {serviceName} + {/* Reads the same bundle atom key as the env switcher, so + it shares that fetch instead of adding a round-trip. */} + } > @@ -404,6 +421,14 @@ function OverviewTab({ const chartBuckets = useMemo(() => detailPoints.map((point) => String(point.bucket)), [detailPoints]) const commitMarkers = useCommitMarkers(releases, chartBuckets) + // An Apdex score means nothing without the target T it was scored against, + // and T is no longer a constant — it follows the service's app kind (500 ms + // for a backend, 2.5 s for a browser app). So the active target is stated on + // the card, in the header slot the grid already provides. + const apdexThresholdMs = Result.builder(overviewResult) + .onSuccess((response) => response.apdexThresholdMs) + .orElse(() => DEFAULT_APDEX_THRESHOLD_MS) + // Stable identity so the memoized chart components under MetricsGrid skip // rerenders when this tab rerenders for unrelated reasons (sibling panel // atoms settling, root-level churn). @@ -419,8 +444,17 @@ function OverviewTab({ tooltip: chart.tooltip, rateMode: chart.rateMode, isLoading: isDetailLoading, + ...(chart.id === "apdex" + ? { + headerValue: ( + + Target < {formatApdexTarget(apdexThresholdMs)} + + ), + } + : {}), })), - [detailPoints, isDetailLoading], + [detailPoints, isDetailLoading, apdexThresholdMs], ) if (Result.isFailure(overviewResult)) { diff --git a/packages/domain/package.json b/packages/domain/package.json index bc0a373e1..5c8f27913 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -16,6 +16,7 @@ "./primitives": "./src/primitives.ts", "./query-engine": "./src/query-engine.ts", "./recommendations": "./src/recommendations.ts", + "./service-app-kind": "./src/service-app-kind.ts", "./setup-audit": "./src/setup-audit.ts", "./tinybird-project-sync": "./src/tinybird/project-sync.ts", "./warehouse-queries": "./src/warehouse-queries.ts", diff --git a/packages/domain/src/clickhouse/migrations/0015_service_app_kind.ts b/packages/domain/src/clickhouse/migrations/0015_service_app_kind.ts new file mode 100644 index 000000000..9e4c968d6 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0015_service_app_kind.ts @@ -0,0 +1,74 @@ +/** + * Migration 0015 — app-kind signals on `service_platforms_hourly`. + * + * `service_platforms_hourly` already answers "where does this service run" + * (k8s / cloudflare / lambda). It could not answer "what kind of app is this", + * because the only signal it carried for that was `maple.sdk.type` — present + * solely on services instrumented with a Maple SDK. A customer on vanilla OTel + * browser JS was indistinguishable from a backend, which matters because the + * service-detail Apdex threshold is derived from the app kind: 500 ms is a + * backend target and scores a browser app as permanently frustrated. + * + * The three added columns are the vendor-neutral markers: + * - `telemetry.sdk.language` — `webjs` is the OTel browser SDK; `swift` / + * `kotlin` the mobile ones. + * - `browser.platform` — per OTel semconv, only ever set in a browser. + * - `device.type` — the mobile-side counterpart. + * + * Every column on this table is `SimpleAggregateFunction(max, String)`, where + * empty sorts first, so a non-empty value from any span in the hour wins the + * merge — "did *any* span carry this attribute", which is the question the + * classifier asks. + * + * **No backfill.** The obvious one is safe (`max` is idempotent, so + * re-inserting a group merges cleanly) but pointless: the classifier reads + * `max()` across the viewed window, so a single hour of post-migration traffic + * classifies the service correctly for every window that includes it. Services + * read `unknown` for at most one hour, and `unknown` already falls back to the + * 500 ms default — the behaviour it has today. A backfill would also have to + * insert `SpanCount = 0` to avoid double-counting the one `sum` column on the + * table, which is a sharp edge with nothing on the other side of it. + * + * `requiredForIngest: false`: nothing on the ingest path changes shape. + * `service_platforms_hourly` is filled by a materialized view, never by a + * native INSERT, so a BYO cluster still running the old view keeps ingesting + * correctly — it simply leaves the three new columns empty, which classifies + * its services exactly as they classify today. Gating ingest on this would + * route every BYO org back to managed over a display-only classification. + */ +export const migration_0015_service_app_kind = { + version: 15, + description: + "Add telemetry.sdk.language / browser.platform / device.type app-kind signals to service_platforms_hourly", + statements: [ + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS TelemetrySdkLanguage SimpleAggregateFunction(max, String) AFTER ProcessRuntimeName", + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS BrowserPlatform SimpleAggregateFunction(max, String) AFTER TelemetrySdkLanguage", + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS DeviceType SimpleAggregateFunction(max, String) AFTER BrowserPlatform", + "DROP VIEW IF EXISTS service_platforms_hourly_mv", + `CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage, + max(ResourceAttributes['browser.platform']) AS BrowserPlatform, + max(ResourceAttributes['device.type']) AS DeviceType, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv`, + ], + requiredForIngest: false, +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 5bfbe9183..7cbd3579e 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -17,6 +17,7 @@ import { migration_0011_session_analytics_columns } from "./0011_session_analyti import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events, webEventsBackfill } from "./0014_web_events" +import { migration_0015_service_app_kind } from "./0015_service_app_kind" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -31,15 +32,44 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) - expect(migrations.at(-1)).toBe(migration_0014_web_events) - expect(latestMigrationVersion).toBe(14) - // 0010 and 0014 are performance-only, so the ingest-gating version skips - // both and stays at 13 — nothing writes `web_events` directly, and bumping - // it would un-ready every BYO-CH org's ingest routing for a read-path change. + expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + expect(migrations.at(-1)).toBe(migration_0015_service_app_kind) + expect(latestMigrationVersion).toBe(15) + // 0010, 0014 and 0015 are read-path-only, so the ingest-gating version skips + // all three and stays at 13 — nothing writes `web_events` or + // `service_platforms_hourly` directly, and bumping it would un-ready every + // BYO-CH org's ingest routing for a read-path change. expect(clickHouseSchemaVersion).toBe("13") expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) + expect(migration_0015_service_app_kind.requiredForIngest).toBe(false) + }) + + it("appends the app-kind signal columns without rewriting service_platforms_hourly", () => { + const sql = migration_0015_service_app_kind.statements.filter((stmt) => !isBackfill(stmt)).join("\n") + + // The vendor-neutral markers: `maple.sdk.type` alone only ever classifies + // services instrumented with a Maple SDK. + expect(sql).toContain( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS TelemetrySdkLanguage", + ) + expect(sql).toContain("max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage") + expect(sql).toContain("max(ResourceAttributes['browser.platform']) AS BrowserPlatform") + expect(sql).toContain("max(ResourceAttributes['device.type']) AS DeviceType") + + // The table already exists, so the view has to be swapped for the columns + // to ever be written — but nothing existing is dropped or rewritten. + expect(sql).toContain("DROP VIEW IF EXISTS service_platforms_hourly_mv") + expect(sql).toContain( + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly", + ) + expect(sql).not.toContain("DROP TABLE") + expect(sql).not.toContain("POPULATE") + + // No backfill: `max()` over the viewed window means one hour of fresh + // telemetry classifies the service, and the only `sum` column on the table + // (SpanCount) would double-count if re-inserted. + expect(migration_0015_service_app_kind.statements.some(isBackfill)).toBe(false) }) it("installs web_events with a live-write MV and no POPULATE", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index b719e9d98..9e62cedfe 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -13,6 +13,7 @@ import { migration_0011_session_analytics_columns } from "./0011_session_analyti import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events } from "./0014_web_events" +import { migration_0015_service_app_kind } from "./0015_service_app_kind" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -58,6 +59,7 @@ export const migrations: ReadonlyArray = [ migration_0012_session_event_attribute_keys, migration_0013_service_map_ingest_bridge, migration_0014_web_events, + migration_0015_service_app_kind, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 8db9cd4ff..9a377596e 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" as const +export const projectRevision = "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -30,7 +30,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS service_operations_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n SpanName String,\n SpanCount SimpleAggregateFunction(sum, UInt64),\n EstimatedSpanCount SimpleAggregateFunction(sum, Float64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationSum SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Minute)\nORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName)\nTTL toDate(Minute) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS service_overview_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ServiceNamespace LowCardinality(String),\n CommitSha LowCardinality(String),\n SpanCount SimpleAggregateFunction(sum, UInt64),\n EstimatedSpanCount SimpleAggregateFunction(sum, Float64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationSum SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64),\n ApdexToleratingCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Hour)\nORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_overview_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", - "CREATE TABLE IF NOT EXISTS service_platforms_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS service_platforms_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n TelemetrySdkLanguage SimpleAggregateFunction(max, String),\n BrowserPlatform SimpleAggregateFunction(max, String),\n DeviceType SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_usage (\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (OrgId, ServiceName, Hour)\nTTL Hour + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS session_events (\n OrgId LowCardinality(String),\n SessionId String,\n Timestamp DateTime64(9),\n Seq UInt32 DEFAULT 0,\n Type LowCardinality(String),\n Url String DEFAULT '',\n TraceId String DEFAULT '',\n Level LowCardinality(String) DEFAULT '',\n Message String DEFAULT '',\n TargetSelector String DEFAULT '',\n TargetText String DEFAULT '',\n NetMethod LowCardinality(String) DEFAULT '',\n NetUrl String DEFAULT '',\n NetStatus UInt16 DEFAULT 0,\n NetDurationMs UInt32 DEFAULT 0,\n ErrorStack String DEFAULT '',\n Attributes Map(String, String),\n INDEX idx_type Type TYPE set(16) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, Timestamp, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replay_events (\n OrgId LowCardinality(String),\n SessionId String,\n ChunkSeq UInt32,\n Timestamp DateTime64(9),\n DurationMs UInt32 DEFAULT 0,\n EventCount UInt32 DEFAULT 0,\n ByteSize UInt32 DEFAULT 0,\n Events String,\n IsCheckpoint UInt8 DEFAULT 0\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, ChunkSeq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -64,7 +64,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles\n FROM traces\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage,\n max(ResourceAttributes['browser.platform']) AS BrowserPlatform,\n max(ResourceAttributes['device.type']) AS DeviceType,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 82957e5e2..0eefa0b95 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" as const +export const projectRevision = "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a" as const export const datasources = [ { @@ -137,7 +137,7 @@ export const datasources = [ { name: "service_platforms_hourly", content: - 'DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) for the service map\'s hosting-icon resolver. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 365 DAY"', + 'DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) plus app-kind signals (telemetry.sdk.language, browser.platform, device.type) for the service map\'s hosting-icon resolver and the service-detail app-kind classifier. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n TelemetrySdkLanguage SimpleAggregateFunction(max, String),\n BrowserPlatform SimpleAggregateFunction(max, String),\n DeviceType SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 365 DAY"', }, { name: "service_usage", @@ -310,7 +310,7 @@ export const pipes = [ { name: "service_platforms_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", + "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) and app-kind signals (telemetry.sdk.language, browser.platform, device.type) into hourly buckets for the service map's runtime-icon resolver and the app-kind classifier.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage,\n max(ResourceAttributes['browser.platform']) AS BrowserPlatform,\n max(ResourceAttributes['device.type']) AS DeviceType,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", }, { name: "service_usage_logs_mv", diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 9018341f9..313b08149 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -18,6 +18,7 @@ import { QueryEngineExecuteResponse, TinybirdDateTime, } from "../query-engine" +import { ServiceAppKind } from "../service-app-kind" import { Authorization } from "./current-tenant" import { warehouseHttpErrors } from "./warehouse" @@ -654,6 +655,28 @@ export class ServiceDetailOverviewResponse extends Schema.Class( @@ -747,7 +770,10 @@ export class ServicePlatformsRequest extends Schema.Class): ServiceAppKindSignals => ({ + ...NO_SIGNALS, + ...overrides, +}) + +describe("classifyServiceAppKind", () => { + const cases: ReadonlyArray = [ + // Maple's own browser SDK writes "browser" (packages/browser/src/tracing.ts). + // It classified as `unknown` before this signal existed, which is exactly how + // every browser app ended up scored against a 500 ms backend target. + ["maple browser SDK", signals({ mapleSdkType: "browser" }), "browser"], + ["maple effect client SDK", signals({ mapleSdkType: "client" }), "browser"], + // The case the vendor-neutral signals exist for: no Maple SDK anywhere. + ["vanilla OTel web", signals({ telemetrySdkLanguage: "webjs" }), "browser"], + ["browser.platform alone", signals({ browserPlatform: "macOS" }), "browser"], + ["maple mobile SDK", signals({ mapleSdkType: "mobile" }), "mobile"], + ["swift SDK", signals({ telemetrySdkLanguage: "swift" }), "mobile"], + ["device.type alone", signals({ deviceType: "phone" }), "mobile"], + ["kubernetes pod", signals({ k8sPodName: "api-7d9f-x2k" }), "backend"], + [ + "cloudflare worker", + signals({ cloudPlatform: "cloudflare.workers", cloudProvider: "cloudflare" }), + "backend", + ], + ["lambda", signals({ faasName: "checkout-handler" }), "backend"], + ["maple server SDK", signals({ mapleSdkType: "server" }), "backend"], + ["no signals at all", NO_SIGNALS, "unknown"], + ] + + for (const [name, input, expected] of cases) { + it(`classifies ${name} as ${expected}`, () => { + expect(classifyServiceAppKind(input)).toBe(expected) + }) + } + + // A browser app can pick up `cloud.provider` from a CDN or a `k8s.*` leak from + // an OTel gateway it was proxied through. Neither makes it a backend, and the + // reverse mistake is impossible — a server never reports `browser.platform`. + it("prefers browser over host-infrastructure signals on the same service", () => { + expect( + classifyServiceAppKind( + signals({ + browserPlatform: "Windows", + cloudProvider: "cloudflare", + k8sPodName: "otel-gateway-abc", + }), + ), + ).toBe("browser") + }) + + it("prefers mobile over host-infrastructure signals on the same service", () => { + expect(classifyServiceAppKind(signals({ mapleSdkType: "mobile", cloudProvider: "aws" }))).toBe( + "mobile", + ) + }) +}) + +describe("apdexThresholdMsForAppKind", () => { + it("keeps the backend default for backend and unknown", () => { + expect(apdexThresholdMsForAppKind("backend")).toBe(DEFAULT_APDEX_THRESHOLD_MS) + expect(apdexThresholdMsForAppKind("unknown")).toBe(DEFAULT_APDEX_THRESHOLD_MS) + }) + + // 2500 ms is the Core Web Vitals "good" LCP boundary, which puts the + // frustrated line (4T, per the Apdex spec) at 10s. + it("scores a browser app against the Core Web Vitals boundary", () => { + expect(apdexThresholdMsForAppKind("browser")).toBe(2500) + }) + + it("raises the mobile target above backend but below browser", () => { + expect(APDEX_THRESHOLD_MS_BY_APP_KIND.mobile).toBeGreaterThan(DEFAULT_APDEX_THRESHOLD_MS) + expect(APDEX_THRESHOLD_MS_BY_APP_KIND.mobile).toBeLessThan(APDEX_THRESHOLD_MS_BY_APP_KIND.browser) + }) +}) diff --git a/packages/domain/src/service-app-kind.ts b/packages/domain/src/service-app-kind.ts new file mode 100644 index 000000000..be5a1a269 --- /dev/null +++ b/packages/domain/src/service-app-kind.ts @@ -0,0 +1,115 @@ +import { Schema } from "effect" + +/** + * What kind of application a service *is* — deliberately orthogonal to + * `ServicePlatform`, which says where it *runs*. A browser app has no hosting + * platform; a Kubernetes pod can be a backend or a batch worker. + * + * The distinction earns its keep in one place today: the Apdex threshold. Apdex + * scores a request against a satisfaction target T, and 500 ms is a target for a + * backend API. Applied to a browser app — where a span is a request made from a + * device on someone's home wifi — it scores every real-world user as frustrated + * and the chart stops carrying signal. + */ +export const ServiceAppKind = Schema.Literals(["browser", "mobile", "backend", "unknown"]) +export type ServiceAppKind = Schema.Schema.Type + +/** The Apdex target for a service whose kind could not be determined, and the + * value every caller that has no service in hand (dashboards, ad-hoc queries, + * alert rules) uses. Also the constant baked into + * `service_overview_hourly.ApdexSatisfiedCount` / `ApdexToleratingCount`, which + * is why the rollup path is only valid at exactly this threshold. */ +export const DEFAULT_APDEX_THRESHOLD_MS = 500 + +/** + * Apdex T per app kind. Frustrated starts at 4T in every case (the Apdex spec), + * so these also set the ceilings: 2 s / 10 s for a browser app, 4 s for mobile. + * + * `browser` is 2500 ms because that is the Core Web Vitals "good" LCP boundary — + * the number the rest of the industry already uses for "this page felt fast" — + * and it puts the frustrated line at 10 s. `mobile` sits between the two: a + * native app's network calls are slower than a datacenter's and faster than a + * cold page load. + */ +export const APDEX_THRESHOLD_MS_BY_APP_KIND: Readonly> = Object.freeze({ + browser: 2500, + mobile: 1000, + backend: DEFAULT_APDEX_THRESHOLD_MS, + unknown: DEFAULT_APDEX_THRESHOLD_MS, +}) + +export const apdexThresholdMsForAppKind = (kind: ServiceAppKind): number => + APDEX_THRESHOLD_MS_BY_APP_KIND[kind] + +/** + * The resource-attribute signals the classifier reads, as stored per hour in + * `service_platforms_hourly`. Every field is "" when the attribute was absent. + */ +export interface ServiceAppKindSignals { + /** `browser.platform` — per OTel semconv, only ever set in a browser. */ + readonly browserPlatform: string + /** `telemetry.sdk.language` — `webjs` is the OTel browser SDK. */ + readonly telemetrySdkLanguage: string + /** `maple.sdk.type` — set only by Maple's own SDKs. */ + readonly mapleSdkType: string + /** `device.type` — the mobile-side marker. */ + readonly deviceType: string + readonly cloudPlatform: string + readonly cloudProvider: string + readonly faasName: string + readonly k8sPodName: string + readonly k8sDeploymentName: string +} + +/** `telemetry.sdk.language` values that only a mobile SDK reports. */ +const MOBILE_SDK_LANGUAGES = new Set(["swift", "objc", "kotlin", "android"]) + +/** + * Classify a service from its resource attributes, first match wins. + * + * Browser is checked before everything else on purpose: a browser app can carry + * `cloud.provider` (a CDN-injected attribute) or a `k8s.*` leak from an OTel + * gateway it was proxied through, and neither makes it a backend. The reverse + * mistake is not possible — a server never reports `browser.platform`. + * + * All-empty signals return `unknown` rather than guessing `backend`: `unknown` + * and `backend` resolve to the same 500 ms threshold, so the honest answer costs + * nothing, and the UI can decline to render a badge it isn't sure about. + */ +export const classifyServiceAppKind = (signals: ServiceAppKindSignals): ServiceAppKind => { + if ( + signals.browserPlatform !== "" || + signals.telemetrySdkLanguage === "webjs" || + signals.mapleSdkType === "browser" || + signals.mapleSdkType === "client" + ) { + return "browser" + } + if ( + signals.mapleSdkType === "mobile" || + MOBILE_SDK_LANGUAGES.has(signals.telemetrySdkLanguage) || + signals.deviceType !== "" + ) { + return "mobile" + } + if ( + signals.k8sPodName !== "" || + signals.k8sDeploymentName !== "" || + signals.cloudPlatform !== "" || + signals.cloudProvider !== "" || + signals.faasName !== "" || + signals.mapleSdkType !== "" + ) { + return "backend" + } + return "unknown" +} + +/** Human label for the app-kind badge. `unknown` has none — the UI renders + * nothing rather than a badge that says it doesn't know. */ +export const SERVICE_APP_KIND_LABELS: Readonly, string>> = + Object.freeze({ + browser: "Browser", + mobile: "Mobile", + backend: "Backend", + }) diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 048fc4b7f..bae05f1cf 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -679,11 +679,18 @@ export type ServiceAddressResolutionsHourlyRow = InferRow { expect(sql).toContain("quantilesTDigestMerge(0.5, 0.95, 0.99)(bDurationQuantiles)") }) + // The service-detail Overview tab is exactly the request above, and it must + // stay that way. `service_overview_hourly.ApdexSatisfiedCount` is computed at + // a hardcoded 500 ms, so a kind-aware Apdex target cannot be threaded through + // this request — it would drop throughput, latency, AND error rate onto the + // 30-day raw path for the sake of one series. The handler re-scores Apdex with + // a second, narrower query instead; this asserts the fork it depends on. + it("drops the annual rollup when a non-default apdex threshold is requested", () => { + const base = { + metric: "count" as const, + needsSampling: true, + allMetrics: true, + rootOnly: true, + bucketSeconds: 3600, + serviceName: "api", + } + expect(canUseAnnualServiceOverview(base)).toBe(true) + expect(canUseAnnualServiceOverview({ ...base, apdexThresholdMs: 500 })).toBe(true) + // A browser service's 2500 ms target. + expect(canUseAnnualServiceOverview({ ...base, apdexThresholdMs: 2500 })).toBe(false) + + const { sql } = compileCH(tracesTimeseriesQuery({ ...base, apdexThresholdMs: 2500 }), baseParams) + expect(sql).not.toContain("FROM service_overview_hourly") + }) + // `service_overview_spans` stores only entry-point spans (Server/Consumer OR // root). Routing an all-spans query there silently swaps the population, which // is how one dashboard showed two answers to the same question. diff --git a/packages/query-engine/src/ch/queries/service-map.test.ts b/packages/query-engine/src/ch/queries/service-map.test.ts index c272b486b..16b1a20e6 100644 --- a/packages/query-engine/src/ch/queries/service-map.test.ts +++ b/packages/query-engine/src/ch/queries/service-map.test.ts @@ -824,6 +824,9 @@ describe("servicePlatformsSQL", () => { faasName: "", mapleSdkType: "node", processRuntimeName: "nodejs", + telemetrySdkLanguage: "nodejs", + browserPlatform: "", + deviceType: "", }, ]) @@ -832,6 +835,8 @@ describe("servicePlatformsSQL", () => { k8sDeploymentName: "artifacts-api", cloudProvider: "aws", mapleSdkType: "node", + telemetrySdkLanguage: "nodejs", + browserPlatform: "", }) }), ) @@ -858,4 +863,22 @@ describe("servicePlatformsSQL", () => { expect(Exit.isFailure(exit)).toBe(true) }), ) + + it("selects the app-kind signal columns", () => { + const sql = servicePlatformsSQL({}, baseParams).sql + + // `maple.sdk.type` alone only classifies services on a Maple SDK; these + // three are what let the classifier see a vanilla-OTel browser or mobile app. + expect(sql).toContain("TelemetrySdkLanguage") + expect(sql).toContain("BrowserPlatform") + expect(sql).toContain("DeviceType") + }) + + it("narrows to one service when asked", () => { + const all = servicePlatformsSQL({}, baseParams).sql + const one = servicePlatformsSQL({ serviceName: "artifacts-api" }, baseParams).sql + + expect(all).not.toContain("artifacts-api") + expect(one).toContain("artifacts-api") + }) }) diff --git a/packages/query-engine/src/ch/queries/service-map.ts b/packages/query-engine/src/ch/queries/service-map.ts index 3d8cffcd6..051af5746 100644 --- a/packages/query-engine/src/ch/queries/service-map.ts +++ b/packages/query-engine/src/ch/queries/service-map.ts @@ -1189,6 +1189,8 @@ export function serviceExternalEdgesSQL( export interface ServicePlatformsOpts { deploymentEnv?: string + /** Narrow to one service — the service-detail app-kind lookup. */ + serviceName?: string } export interface ServicePlatformsOutput { @@ -1201,6 +1203,9 @@ export interface ServicePlatformsOutput { readonly faasName: string readonly mapleSdkType: string readonly processRuntimeName: string + readonly telemetrySdkLanguage: string + readonly browserPlatform: string + readonly deviceType: string } const ServicePlatformsOutputSchema: CompiledQueryRowSchema = Schema.Struct({ @@ -1213,6 +1218,9 @@ const ServicePlatformsOutputSchema: CompiledQueryRowSchema [ $.OrgId.eq(param.string("orgId")), $.Hour.gte(CH.toStartOfHour(CH.toDateTime(param.dateTime("startTime")))), $.Hour.lte(param.dateTime("endTime")), $.ServiceName.neq(""), + opts.serviceName ? $.ServiceName.eq(opts.serviceName) : undefined, opts.deploymentEnv ? $.DeploymentEnv.eq(opts.deploymentEnv) : undefined, ]) .groupBy("serviceName") diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 998ce8763..38bf28c30 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -480,6 +480,9 @@ export const ServicePlatformsHourly = table("service_platforms_hourly", { FaasName: T.string, MapleSdkType: T.string, ProcessRuntimeName: T.string, + TelemetrySdkLanguage: T.string, + BrowserPlatform: T.string, + DeviceType: T.string, SpanCount: T.uint64, }) diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index 0f956bc40..b1f5f8b71 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -543,6 +543,29 @@ export const servicePlatforms = defineQuery({ ), }) +/** + * The same platform/app-kind row for a single service — the service-detail + * lookup that picks the page's Apdex threshold. + * + * Cached for 5 minutes rather than the usual 15 seconds: a service's app kind + * is a property of how it is instrumented, so it changes on deploy at most, and + * this read sits in front of the Overview tab's timeseries. On a cold key it + * costs one small aggregate-table scan; on every other load it costs nothing. + */ +export const serviceAppKind = defineQuery({ + id: "serviceAppKind", + profile: "aggregation", + cache: 300, + compile: ( + payload: { serviceName: string; startTime: string; endTime: string; deploymentEnv?: string }, + orgId: string, + ) => + CH.servicePlatformsSQL( + { serviceName: payload.serviceName, deploymentEnv: payload.deploymentEnv }, + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + const dbQueryParams = (payload: ServiceDbQuerySummaryRequest, orgId: string) => ({ orgId, dbSystem: payload.dbSystem, diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index a084f9030..328c60c6a 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -16,6 +16,9 @@ import { LOCAL_SCHEMA_V4, LOCAL_SCHEMA_V4_MANIFEST_DIGEST, LOCAL_SCHEMA_V4_SQL, + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST_DIGEST, + LOCAL_SCHEMA_V5_SQL, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -104,6 +107,18 @@ if ( fail("the immutable local schema v4 snapshot no longer matches its historical identity") } +const v5 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V5.version) +if ( + !v5 || + LOCAL_SCHEMA_V5_MANIFEST_DIGEST !== v5.manifestDigest || + schemaFingerprint(LOCAL_SCHEMA_V5_SQL) !== v5.fingerprint || + schemaDigest(LOCAL_SCHEMA_V5_SQL) !== v5.digest || + LOCAL_SCHEMA_V5.fingerprint !== v5.fingerprint || + LOCAL_SCHEMA_V5.digest !== v5.digest +) { + fail("the immutable local schema v5 snapshot no longer matches its historical identity") +} + const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name) if (new Set(names).size !== names.length) fail("local structural schema manifest contains duplicate object names")