diff --git a/apps/api/src/routes/v1/query-engine.http.ts b/apps/api/src/routes/v1/query-engine.http.ts index dceeab780..ecc70e61e 100644 --- a/apps/api/src/routes/v1/query-engine.http.ts +++ b/apps/api/src/routes/v1/query-engine.http.ts @@ -71,6 +71,12 @@ import { TraceId, SpanId, } from "@maple/domain/http" +import { + apdexThresholdMsForAppKind, + classifyServiceAppKind, + DEFAULT_APDEX_THRESHOLD_MS, + type ServiceAppKind, +} from "@maple/domain/service-app-kind" import { Clock, Effect, Match, Option, Schema } from "effect" import { QueryEngineService } from "@/services/warehouse/QueryEngineService" import { makeDirectRouteCachePolicy, makeExecuteRawSql } from "@maple/query-engine/runtime" @@ -233,9 +239,16 @@ const toServicePlatformRow = (row: CH.ServicePlatformsOutput) => { const faasName = String(row.faasName ?? "") const mapleSdkType = String(row.mapleSdkType ?? "") const processRuntimeName = String(row.processRuntimeName ?? "") + // App-kind signals. Optional on the row so a query compiled before migration + // 0015 (or a cluster that has not applied it) decodes as "no signal". + const telemetrySdkLanguage = String(row.telemetrySdkLanguage ?? "") + const browserPlatform = String(row.browserPlatform ?? "") + const deviceType = String(row.deviceType ?? "") // cluster.name alone does not prove the service runs in Kubernetes. const isKubernetes = k8sPodName !== "" || k8sDeploymentName !== "" - // Host infrastructure takes precedence over SDK self-report. + // Host infrastructure takes precedence over SDK self-report. `browser` is + // what Maple's own browser SDK reports (packages/browser); `client` is the + // Effect client SDK. const platform: "kubernetes" | "cloudflare" | "lambda" | "web" | "unknown" = cloudPlatform === "cloudflare.workers" || cloudProvider === "cloudflare" ? "cloudflare" @@ -243,12 +256,23 @@ const toServicePlatformRow = (row: CH.ServicePlatformsOutput) => { ? "lambda" : isKubernetes ? "kubernetes" - : mapleSdkType === "client" + : mapleSdkType === "client" || mapleSdkType === "browser" ? "web" : "unknown" return { serviceName: decodeServiceName(String(row.serviceName ?? "")), platform, + appKind: classifyServiceAppKind({ + browserPlatform, + telemetrySdkLanguage, + mapleSdkType, + deviceType, + cloudPlatform, + cloudProvider, + faasName, + k8sPodName, + k8sDeploymentName, + }), k8sCluster, cloudPlatform, cloudProvider, @@ -922,7 +946,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", // execute-path cache; releases is uncached (mirrors the standalone // handler); environments is edge-cached on a service-scoped key. yield* warehouse.warmRoute(tenant) - const [timeseries, releaseRows, environmentRows] = yield* Effect.all( + const [timeseries, releaseRows, environmentRows, appKindRows] = yield* Effect.all( [ queryEngine.execute(tenant, payload.timeseries), runQuery(Queries.serviceReleases, tenant, payload), @@ -931,9 +955,51 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", startTime: payload.startTime, endTime: payload.endTime, }), + // What kind of app this is, which is what picks the Apdex + // target below. Runs alongside the rest — it gates only the + // optional override query, not the primary chart. + runQuery(Queries.serviceAppKind, tenant, { + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + }), ], - { concurrency: 3 }, + { concurrency: 4 }, ) + + const appKind: ServiceAppKind = + appKindRows.length > 0 ? toServicePlatformRow(appKindRows[0]!).appKind : "unknown" + const apdexThresholdMs = apdexThresholdMsForAppKind(appKind) + + // `payload.timeseries` is forwarded untouched so it keeps the + // annual service-overview rollup, whose stored Apdex counters are + // baked at 500 ms (`canUseAnnualServiceOverview` enforces that). + // Threading a different threshold through it would knock + // throughput, latency, AND error rate onto the 30-day raw path for + // the sake of one series — so a non-default target is re-scored by + // this second, narrower query instead. + const apdexOverride = + apdexThresholdMs === DEFAULT_APDEX_THRESHOLD_MS + ? undefined + : yield* runQuery(Queries.serviceApdex, tenant, { + serviceName: payload.serviceName, + startTime: payload.startTime, + endTime: payload.endTime, + bucketSeconds: + payload.timeseries.query.kind === "timeseries" + ? payload.timeseries.query.bucketSeconds + : payload.releasesBucketSeconds, + apdexThresholdMs, + }).pipe( + Effect.map((rows) => + rows.map((row) => ({ + bucket: String(row.bucket), + apdexScore: Number(row.apdexScore), + totalCount: Number(row.totalCount), + })), + ), + ) + return new ServiceDetailOverviewResponse({ timeseries, releases: releaseRows.map((row) => ({ @@ -945,6 +1011,9 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", environments: environmentRows .map((row) => String(row.environment ?? "")) .filter((env) => env !== ""), + appKind, + apdexThresholdMs, + ...(apdexOverride === undefined ? {} : { apdexOverride }), }) }), ) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index c5d051f39..60cc6ba81 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -51,4 +51,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "826f9363db5dd7722debd0c87a5b74a5b66387f4752abc219d4cc0ce76358a9e", projectRevision: "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a", }), + Object.freeze({ + version: 5, + fingerprint: "3099929d42b2ce8b", + digest: "3099929d42b2ce8b18c06a428a8e32a51ce9724300138241110e08a3f09e8193", + manifestDigest: "99d834ae3baab1d0a753f18a96b96a0130bb0af8964a0b119f1f4203e3bc6d0f", + projectRevision: "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 19b611ed5..40530abf3 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 4 as const +export const LOCAL_SCHEMA_VERSION = 5 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index aa702a8e6..16d3867e6 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -36,6 +36,7 @@ import { legacyToCurrentModule } from "./local-store-migrations/legacy-to-curren import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error-rollup" import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" +import { v4ToV5ServiceAppKindModule } from "./local-store-migrations/v4-to-v5-service-app-kind" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -119,6 +120,7 @@ export const localStoreMigrations: ReadonlyArray = v1ToV2ErrorRollupModule, v2ToV3ServiceMapIngestBridgeModule, v3ToV4WebEventsModule, + v4ToV5ServiceAppKindModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-app-kind.ts b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-app-kind.ts new file mode 100644 index 000000000..3f6f04290 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-app-kind.ts @@ -0,0 +1,246 @@ +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V4, + LOCAL_SCHEMA_V4_MANIFEST, + LOCAL_SCHEMA_V4_SQL, + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V5_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +interface V4ToV5State { + readonly module: "local-0004-to-0005-service-app-kind" + readonly version: 1 + readonly rawRows: Readonly> + readonly retentionDays?: number +} + +interface V4ToV5Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v4 -> v5 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (typeof count !== "string" || !/^\d+$/.test(count)) + throw new Error(`v4 -> v5 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v4 -> v5 rawRows contains an unknown table") + return counts +} + +const decodeState = (value: unknown): V4ToV5State => { + if (!isRecord(value)) throw new Error("v4 -> v5 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v4 -> v5 state contains an unknown field") + if (value.module !== "local-0004-to-0005-service-app-kind" || value.version !== 1) + throw new Error("v4 -> v5 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v4 -> v5 retentionDays must be an integer") + return { + module: "local-0004-to-0005-service-app-kind", + version: 1, + rawRows: decodeCounts(value.rawRows), + ...(value.retentionDays === undefined ? {} : { retentionDays: value.retentionDays }), + } +} + +const decodeProgress = (value: unknown): V4ToV5Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v4 -> v5 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V4_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V4_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V4_SQL, bootstrapSchema: false }, + ) + return { + module: "local-0004-to-0005-service-app-kind", + version: 1, + rawRows, + ...(retentionDays === undefined ? {} : { retentionDays }), + } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V4ToV5State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Three appended columns on `service_platforms_hourly` plus the view that fills + * them. Unlike v3 -> v4 the bootstrap pass alone is not enough: the table + * already exists, so its `CREATE TABLE IF NOT EXISTS` is a no-op and the + * columns would never appear. The ALTERs run first, against the v4 snapshot, + * and the view is dropped so the bootstrap recreates it with the widened + * SELECT. + * + * `SimpleAggregateFunction(max, String)` columns default to empty, which is + * exactly what the classifier reads as "no signal" — so historical hours keep + * classifying as they do today (`unknown` -> the 500 ms Apdex default) and + * converge as soon as one hour of fresh telemetry lands. Nothing is rewritten. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS TelemetrySdkLanguage SimpleAggregateFunction(max, String) AFTER ProcessRuntimeName", + ) + db.exec( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS BrowserPlatform SimpleAggregateFunction(max, String) AFTER TelemetrySdkLanguage", + ) + db.exec( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS DeviceType SimpleAggregateFunction(max, String) AFTER BrowserPlatform", + ) + db.exec("DROP VIEW IF EXISTS service_platforms_hourly_mv") + }, + { schemaSql: LOCAL_SCHEMA_V4_SQL, bootstrapSchema: false }, + ) + return context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V5_SQL, + bootstrapSchema: true, + }) +} + +const verify = async ( + context: MigrationModuleContext, + state: V4ToV5State, + _progress: V4ToV5Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V5_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v4 -> v5 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v4-store", + description: "Clone the stopped v4 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "install-app-kind-columns", + description: + "Append the app-kind signal columns to service_platforms_hourly and recreate its materialized view", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v5-schema", + description: "Verify the v5 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v4 store is cloned byte-for-byte before additive DDL runs.", + }, + { + // Appended columns only — no existing column is read, rewritten, or + // reordered, and the pre-existing rows keep every value they had. The new + // columns read as empty for historical hours, which the classifier already + // treats as "no signal" and resolves to the same 500 ms Apdex default those + // hours get today. + name: "service_platforms_hourly", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Existing rows and columns are untouched; the three appended signal columns fill forward from traces writes and are complete for any window containing one hour of post-migration telemetry.", + preservationInterval: "service_platforms_hourly retention horizon", + sourceRetentionDays: 365, + targetRetentionDays: 365, + }, +] + +export const v4ToV5ServiceAppKindModule: LocalStoreMigrationModule = { + id: "local-0004-to-0005-service-app-kind", + moduleVersion: 1, + description: + "Append telemetry.sdk.language / browser.platform / device.type app-kind signals to service_platforms_hourly", + from: LOCAL_SCHEMA_V4, + to: LOCAL_SCHEMA_V5, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index a3ba0c12a..c7105cf2c 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -3,6 +3,7 @@ import schemaV1Sql from "./schema/local-schema-v1.sql" with { type: "text" } import schemaV2Sql from "./schema/local-schema-v2.sql" with { type: "text" } import schemaV3Sql from "./schema/local-schema-v3.sql" with { type: "text" } import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } +import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -26,7 +27,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" + "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -57,6 +58,11 @@ export const LOCAL_SCHEMA_V3_MANIFEST_DIGEST = LOCAL_SCHEMA_V3_MANIFEST.digest export const LOCAL_SCHEMA_V4_SQL = schemaV4Sql export const LOCAL_SCHEMA_V4_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV4Sql) export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest +/** Immutable v5 DDL/manifest snapshot used by the v4 -> v5 module after the + * generated current schema advances. */ +export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql +export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) +export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -107,6 +113,15 @@ export const LOCAL_SCHEMA_V4: LocalSchemaIdentity = Object.freeze({ projectRevision: LOCAL_SCHEMA_HISTORY[4]!.projectRevision, }) +export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_HISTORY[5]!.version, + fingerprint: LOCAL_SCHEMA_HISTORY[5]!.fingerprint, + digest: LOCAL_SCHEMA_HISTORY[5]!.digest, + manifestDigest: LOCAL_SCHEMA_HISTORY[5]!.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, +}) + export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, fingerprint: SCHEMA_FINGERPRINT, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index cebc793d3..f3cc2a508 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a", + "projectRevision": "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v5.sql b/apps/cli/src/server/schema/local-schema-v5.sql new file mode 100644 index 000000000..4f756e503 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v5.sql @@ -0,0 +1,1702 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a +-- localSchemaVersion: 5 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + TelemetrySdkLanguage SimpleAggregateFunction(max, String), + BrowserPlatform SimpleAggregateFunction(max, String), + DeviceType SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage, + max(ResourceAttributes['browser.platform']) AS BrowserPlatform, + max(ResourceAttributes['device.type']) AS DeviceType, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 59ffa044f..4f756e503 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a --- localSchemaVersion: 4 +-- projectRevision: 7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a +-- localSchemaVersion: 5 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -568,6 +568,9 @@ CREATE TABLE IF NOT EXISTS service_platforms_hourly ( FaasName SimpleAggregateFunction(max, String), MapleSdkType SimpleAggregateFunction(max, String), ProcessRuntimeName SimpleAggregateFunction(max, String), + TelemetrySdkLanguage SimpleAggregateFunction(max, String), + BrowserPlatform SimpleAggregateFunction(max, String), + DeviceType SimpleAggregateFunction(max, String), SpanCount SimpleAggregateFunction(sum, UInt64) ) ENGINE = AggregatingMergeTree @@ -1416,6 +1419,9 @@ SELECT max(ResourceAttributes['faas.name']) AS FaasName, max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage, + max(ResourceAttributes['browser.platform']) AS BrowserPlatform, + max(ResourceAttributes['device.type']) AS DeviceType, count() AS SpanCount FROM traces WHERE ServiceName != '' diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 2df59982e..3b27fa4ba 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -12,6 +12,8 @@ import { LOCAL_SCHEMA_V3, LOCAL_SCHEMA_V3_MANIFEST, LOCAL_SCHEMA_V4, + LOCAL_SCHEMA_V4_MANIFEST, + LOCAL_SCHEMA_V5, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -53,16 +55,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v4 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("75ac856927d88d56") - expect(SCHEMA_DIGEST).toBe("75ac856927d88d56518f12c68407a8f2a199d000b6eeb8576f9c97000138f5a4") + it("matches the generated v5 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("3099929d42b2ce8b") + expect(SCHEMA_DIGEST).toBe("3099929d42b2ce8b18c06a428a8e32a51ce9724300138241110e08a3f09e8193") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(4) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V4) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(5) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V5) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -85,7 +87,7 @@ describe("current local schema identity", () => { ) expect(v2TimeOrderedErrors?.definition).toContain("FROM error_events") - // v4 is exactly v3 plus the web analytics fact table and its view. Asserted + // v4 was exactly v3 plus the web analytics fact table and its view. Asserted // against the frozen v3 manifest rather than the diff so a later structural // change can't quietly ride along on this version. const webEvents = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "web_events") @@ -99,6 +101,22 @@ describe("current local schema identity", () => { expect( LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name).filter((name) => !v3Names.has(name)), ).toEqual(["web_events", "web_events_mv"]) + + // v5 adds no objects at all — only three app-kind signal columns on an + // existing table. Asserting the object sets are identical is what stops a + // structural change riding along on a column-only version. + expect(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)).toEqual( + LOCAL_SCHEMA_V4_MANIFEST.objects.map((object) => object.name), + ) + const appKindColumns = ["TelemetrySdkLanguage", "BrowserPlatform", "DeviceType"] + const platformsColumnsOf = (manifest: LocalSchemaManifest) => + manifest.objects + .find((object) => object.name === "service_platforms_hourly") + ?.columns.map((column) => column.name) ?? [] + expect(platformsColumnsOf(LOCAL_SCHEMA_MANIFEST)).toEqual(expect.arrayContaining(appKindColumns)) + expect(platformsColumnsOf(LOCAL_SCHEMA_V4_MANIFEST)).not.toEqual( + expect.arrayContaining(appKindColumns), + ) }) }) @@ -110,12 +128,14 @@ describe("local migration registry", () => { "local-0001-to-0002-error-rollup", "local-0002-to-0003-service-map-ingest-bridge", "local-0003-to-0004-web-events", + "local-0004-to-0005-service-app-kind", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) expect(chain[1]?.to).toEqual(LOCAL_SCHEMA_V2) expect(chain[2]?.to).toEqual(LOCAL_SCHEMA_V3) expect(chain[3]?.to).toEqual(LOCAL_SCHEMA_V4) + expect(chain[4]?.to).toEqual(LOCAL_SCHEMA_V5) expect(typeof chain[0]?.apply).toBe("function") }) @@ -151,7 +171,7 @@ describe("local migration registry", () => { ).toThrow(/no registered/) expect(() => resolveMigrationChain( - { ...CURRENT_LOCAL_SCHEMA, version: 5, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 6, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 08ff5ec91..a97f777f3 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a"; +pub const PROJECT_REVISION: &str = "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/web/src/api/warehouse/custom-charts.test.ts b/apps/web/src/api/warehouse/custom-charts.test.ts index 2ec711a1d..a48f3c669 100644 --- a/apps/web/src/api/warehouse/custom-charts.test.ts +++ b/apps/web/src/api/warehouse/custom-charts.test.ts @@ -16,6 +16,7 @@ import { fillServiceDetailPoints, getCustomChartServiceDetail, getServiceDetailThroughputRefinement, + mergeApdexOverride, mergeExactThroughput, } from "@/api/warehouse/custom-charts" import type { ServiceDetailTimeSeriesPoint } from "@/api/warehouse/services" @@ -182,3 +183,48 @@ describe("fillServiceDetailPoints", () => { expect(result.every((p) => p.partial === false)).toBe(true) }) }) + +describe("mergeApdexOverride", () => { + const point = (bucket: string, apdexScore: number | null): ServiceDetailTimeSeriesPoint => ({ + bucket, + throughput: 10, + tracedThroughput: 10, + hasSampling: false, + samplingWeight: 1, + errorRate: 0, + p50LatencyMs: 0, + p95LatencyMs: 0, + p99LatencyMs: 0, + apdexScore, + totalCount: 10, + partial: false, + }) + + it("replaces the 500ms-scored series with the kind-aware one", () => { + const points = [point("2026-02-01T00:00:00.000Z", 0.2), point("2026-02-01T01:00:00.000Z", 0.3)] + const merged = mergeApdexOverride( + points, + new Map([ + ["2026-02-01T00:00:00.000Z", 0.94], + ["2026-02-01T01:00:00.000Z", 0.91], + ]), + ) + + expect(merged.map((p) => p.apdexScore)).toEqual([0.94, 0.91]) + // Only apdex is re-scored — the rest of the point still comes from the + // annual rollup the primary timeseries stayed on. + expect(merged[0].throughput).toBe(10) + }) + + // The override reads `service_overview_spans` (30-day TTL) while the rest of + // the chart reaches a year back. Carrying the 500ms number through for the + // uncovered buckets would mix two thresholds in one series; 0 would draw a + // crater that reads as "every user was frustrated". + it("nulls buckets the override does not cover rather than keeping or zeroing them", () => { + const points = [point("2026-01-01T00:00:00.000Z", 0.2), point("2026-02-01T00:00:00.000Z", 0.3)] + const merged = mergeApdexOverride(points, new Map([["2026-02-01T00:00:00.000Z", 0.88]])) + + expect(merged[0].apdexScore).toBe(null) + expect(merged[1].apdexScore).toBe(0.88) + }) +}) diff --git a/apps/web/src/api/warehouse/custom-charts.ts b/apps/web/src/api/warehouse/custom-charts.ts index e8752464d..76258130d 100644 --- a/apps/web/src/api/warehouse/custom-charts.ts +++ b/apps/web/src/api/warehouse/custom-charts.ts @@ -31,6 +31,7 @@ import { invalidWarehouseInput, runWarehouseQuery, } from "@/api/warehouse/effect-utils" +import { DEFAULT_APDEX_THRESHOLD_MS, type ServiceAppKind } from "@maple/domain/service-app-kind" import { MapleApiAtomClient } from "@/lib/services/common/atom-client" import type { ServiceDetailTimeSeriesPoint, ServiceTimeSeriesPoint } from "@/api/warehouse/services" const dateTimeString = WarehouseDateTimeString @@ -837,6 +838,31 @@ export interface ServiceDetailOverviewResult { data: ServiceDetailTimeSeriesPoint[] releases: ReadonlyArray<{ bucket: string; commitSha: CommitSha; count: number; errorCount: number }> environments: string[] + /** What kind of app this service is — drives the header badge. */ + appKind: ServiceAppKind + /** The Apdex target `data[].apdexScore` was scored against, in ms. Shown on + * the chart: a score is uninterpretable without the T that produced it. */ + apdexThresholdMs: number +} + +/** + * Replace the 500 ms-scored Apdex series with one re-scored at the service's own + * target, as returned by the `serviceDetailOverview` handler. + * + * Buckets the override does not cover become `null`, not 0. The override reads + * `service_overview_spans` (30-day TTL) while the rest of the chart can reach a + * year back, so on a long range the early buckets genuinely have no score — + * carrying the 500 ms number through for those would silently mix two + * thresholds in one series, and zero would draw a crater. + */ +export function mergeApdexOverride( + points: ReadonlyArray, + overrideByBucket: ReadonlyMap, +): ServiceDetailTimeSeriesPoint[] { + return points.map((point) => ({ + ...point, + apdexScore: overrideByBucket.get(point.bucket) ?? null, + })) } export function getServiceDetailOverview({ data }: { data: GetCustomChartServiceDetailInput }) { @@ -879,8 +905,20 @@ const getServiceDetailOverviewEffect = Effect.fn("QueryEngine.getServiceDetailOv }), ) + const points = buildServiceDetailPoints(result.timeseries, startTime, endTime, bucketSeconds, nowMs) + // Present only when the service's app kind sets a target other than the + // 500 ms the primary timeseries is scored at. Absent on an older API build, + // which is the same as "the default applies". + const apdexOverride = result.apdexOverride + return { - data: buildServiceDetailPoints(result.timeseries, startTime, endTime, bucketSeconds, nowMs), + data: + apdexOverride === undefined + ? points + : mergeApdexOverride( + points, + new Map(apdexOverride.map((row) => [toIsoBucket(row.bucket), row.apdexScore])), + ), releases: result.releases.map((r) => ({ bucket: toIsoBucket(r.bucket), commitSha: r.commitSha, @@ -889,6 +927,8 @@ const getServiceDetailOverviewEffect = Effect.fn("QueryEngine.getServiceDetailOv errorCount: Number(r.errorCount ?? 0), })), environments: [...result.environments], + appKind: result.appKind ?? "unknown", + apdexThresholdMs: result.apdexThresholdMs ?? DEFAULT_APDEX_THRESHOLD_MS, } satisfies ServiceDetailOverviewResult }) diff --git a/apps/web/src/api/warehouse/services.ts b/apps/web/src/api/warehouse/services.ts index c8b006fd4..9694d3485 100644 --- a/apps/web/src/api/warehouse/services.ts +++ b/apps/web/src/api/warehouse/services.ts @@ -546,7 +546,13 @@ export interface ServiceDetailTimeSeriesPoint { p50LatencyMs: number p95LatencyMs: number p99LatencyMs: number - apdexScore: number + /** + * `null` where the score is unknown rather than zero. A service whose app + * kind sets a non-default Apdex target is re-scored from + * `service_overview_spans` (30-day TTL), so on a longer range the early + * buckets have no score — and 0 would render as "everyone was frustrated". + */ + apdexScore: number | null totalCount: number /** * The bucket is still settling — its window ends within the ingestion-lag diff --git a/apps/web/src/components/services/service-app-kind-badge.tsx b/apps/web/src/components/services/service-app-kind-badge.tsx new file mode 100644 index 000000000..ae58001e7 --- /dev/null +++ b/apps/web/src/components/services/service-app-kind-badge.tsx @@ -0,0 +1,74 @@ +import { SERVICE_APP_KIND_LABELS, type ServiceAppKind } from "@maple/domain/service-app-kind" +import { Badge } from "@maple/ui/components/ui/badge" +import { cn } from "@maple/ui/lib/utils" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { GlobeIcon, MobileIcon, ServerIcon } from "@/components/icons" +import { getServiceDetailOverviewResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" + +interface ServiceAppKindBadgeProps { + serviceName: string + startTime?: string + endTime?: string + /** Mirrors the Overview tab's bundle-atom input so this shares that fetch + * rather than issuing its own — same contract as + * `ServiceEnvironmentSwitcher`. */ + environments?: string[] + className?: string +} + +// Same token-based palette convention as `DependencyTypeBadge`: every tone maps +// onto an existing chart/severity token so the badge tracks the theme. +const tones: Record, string> = { + browser: "bg-chart-2/10 text-chart-2", + mobile: "bg-chart-4/10 text-chart-4", + backend: "bg-foreground/5 text-muted-foreground", +} + +function getIcon(kind: Exclude) { + switch (kind) { + case "browser": + return GlobeIcon + case "mobile": + return MobileIcon + case "backend": + return ServerIcon + } +} + +/** + * What kind of app this service is, derived from its resource attributes (see + * `classifyServiceAppKind`). It is not decoration: the same classification picks + * the Apdex target the Overview chart is scored against, so the badge is what + * makes that number's basis visible on the page. + * + * Renders nothing for `unknown` — a badge that says the product could not tell + * is worse than no badge, and `unknown` resolves to the same default target as + * `backend` anyway. + */ +export function ServiceAppKindBadge({ + serviceName, + startTime, + endTime, + environments, + className, +}: ServiceAppKindBadgeProps) { + const overviewResult = useAtomValue( + getServiceDetailOverviewResultAtom({ + data: { serviceName, startTime, endTime, environments }, + }), + ) + + const kind = Result.builder(overviewResult) + .onSuccess((response) => response.appKind) + .orElse((): ServiceAppKind => "unknown") + + if (kind === "unknown") return null + + const Icon = getIcon(kind) + return ( + + + {SERVICE_APP_KIND_LABELS[kind]} + + ) +} diff --git a/apps/web/src/routes/services/$serviceName.tsx b/apps/web/src/routes/services/$serviceName.tsx index 217d067d7..cc828ca23 100644 --- a/apps/web/src/routes/services/$serviceName.tsx +++ b/apps/web/src/routes/services/$serviceName.tsx @@ -15,6 +15,7 @@ import { getServiceDetailThroughputRefinementResultAtom, } from "@/lib/services/atoms/warehouse-query-atoms" import { mergeExactThroughput } from "@/api/warehouse/custom-charts" +import { DEFAULT_APDEX_THRESHOLD_MS } from "@maple/domain/service-app-kind" import type { ServiceDetailTimeSeriesPoint } from "@/api/warehouse/services" import { useCommitMarkers } from "@/components/vcs/commit-markers/use-commit-markers" import type { ReleasePoint } from "@/components/vcs/commit-markers/marker-layout" @@ -26,6 +27,7 @@ import { BellIcon } from "@/components/icons" import { ServiceDependenciesTab } from "@/components/services/service-dependencies-tab" import { ServiceOperationsTab } from "@/components/services/service-operations-tab" import { ServiceDependencyStrip } from "@/components/services/service-dependency-strip" +import { ServiceAppKindBadge } from "@/components/services/service-app-kind-badge" import { ServiceEnvironmentSwitcher } from "@/components/services/service-environment-switcher" import { ServiceErrorsPanel } from "@/components/services/service-errors-panel" import { ServiceRecentDeploys } from "@/components/services/service-recent-deploys" @@ -108,6 +110,13 @@ const SERVICE_CHARTS: ServiceChartConfig[] = [ }, ] +/** "500ms" / "2.5s" — sub-second targets stay in ms, the rest read as seconds. */ +function formatApdexTarget(thresholdMs: number): string { + if (thresholdMs < 1000) return `${thresholdMs}ms` + const seconds = thresholdMs / 1000 + return `${Number.isInteger(seconds) ? seconds : seconds.toFixed(1)}s` +} + function ServiceDetailPage() { const search = Route.useSearch() return ( @@ -194,6 +203,14 @@ function ServiceDetailContent() { {serviceName} + {/* Reads the same bundle atom key as the env switcher, so + it shares that fetch instead of adding a round-trip. */} + } > @@ -404,6 +421,14 @@ function OverviewTab({ const chartBuckets = useMemo(() => detailPoints.map((point) => String(point.bucket)), [detailPoints]) const commitMarkers = useCommitMarkers(releases, chartBuckets) + // An Apdex score means nothing without the target T it was scored against, + // and T is no longer a constant — it follows the service's app kind (500 ms + // for a backend, 2.5 s for a browser app). So the active target is stated on + // the card, in the header slot the grid already provides. + const apdexThresholdMs = Result.builder(overviewResult) + .onSuccess((response) => response.apdexThresholdMs) + .orElse(() => DEFAULT_APDEX_THRESHOLD_MS) + // Stable identity so the memoized chart components under MetricsGrid skip // rerenders when this tab rerenders for unrelated reasons (sibling panel // atoms settling, root-level churn). @@ -419,8 +444,17 @@ function OverviewTab({ tooltip: chart.tooltip, rateMode: chart.rateMode, isLoading: isDetailLoading, + ...(chart.id === "apdex" + ? { + headerValue: ( + + Target < {formatApdexTarget(apdexThresholdMs)} + + ), + } + : {}), })), - [detailPoints, isDetailLoading], + [detailPoints, isDetailLoading, apdexThresholdMs], ) if (Result.isFailure(overviewResult)) { diff --git a/packages/domain/package.json b/packages/domain/package.json index bc0a373e1..5c8f27913 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -16,6 +16,7 @@ "./primitives": "./src/primitives.ts", "./query-engine": "./src/query-engine.ts", "./recommendations": "./src/recommendations.ts", + "./service-app-kind": "./src/service-app-kind.ts", "./setup-audit": "./src/setup-audit.ts", "./tinybird-project-sync": "./src/tinybird/project-sync.ts", "./warehouse-queries": "./src/warehouse-queries.ts", diff --git a/packages/domain/src/clickhouse/migrations/0015_service_app_kind.ts b/packages/domain/src/clickhouse/migrations/0015_service_app_kind.ts new file mode 100644 index 000000000..9e4c968d6 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0015_service_app_kind.ts @@ -0,0 +1,74 @@ +/** + * Migration 0015 — app-kind signals on `service_platforms_hourly`. + * + * `service_platforms_hourly` already answers "where does this service run" + * (k8s / cloudflare / lambda). It could not answer "what kind of app is this", + * because the only signal it carried for that was `maple.sdk.type` — present + * solely on services instrumented with a Maple SDK. A customer on vanilla OTel + * browser JS was indistinguishable from a backend, which matters because the + * service-detail Apdex threshold is derived from the app kind: 500 ms is a + * backend target and scores a browser app as permanently frustrated. + * + * The three added columns are the vendor-neutral markers: + * - `telemetry.sdk.language` — `webjs` is the OTel browser SDK; `swift` / + * `kotlin` the mobile ones. + * - `browser.platform` — per OTel semconv, only ever set in a browser. + * - `device.type` — the mobile-side counterpart. + * + * Every column on this table is `SimpleAggregateFunction(max, String)`, where + * empty sorts first, so a non-empty value from any span in the hour wins the + * merge — "did *any* span carry this attribute", which is the question the + * classifier asks. + * + * **No backfill.** The obvious one is safe (`max` is idempotent, so + * re-inserting a group merges cleanly) but pointless: the classifier reads + * `max()` across the viewed window, so a single hour of post-migration traffic + * classifies the service correctly for every window that includes it. Services + * read `unknown` for at most one hour, and `unknown` already falls back to the + * 500 ms default — the behaviour it has today. A backfill would also have to + * insert `SpanCount = 0` to avoid double-counting the one `sum` column on the + * table, which is a sharp edge with nothing on the other side of it. + * + * `requiredForIngest: false`: nothing on the ingest path changes shape. + * `service_platforms_hourly` is filled by a materialized view, never by a + * native INSERT, so a BYO cluster still running the old view keeps ingesting + * correctly — it simply leaves the three new columns empty, which classifies + * its services exactly as they classify today. Gating ingest on this would + * route every BYO org back to managed over a display-only classification. + */ +export const migration_0015_service_app_kind = { + version: 15, + description: + "Add telemetry.sdk.language / browser.platform / device.type app-kind signals to service_platforms_hourly", + statements: [ + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS TelemetrySdkLanguage SimpleAggregateFunction(max, String) AFTER ProcessRuntimeName", + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS BrowserPlatform SimpleAggregateFunction(max, String) AFTER TelemetrySdkLanguage", + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS DeviceType SimpleAggregateFunction(max, String) AFTER BrowserPlatform", + "DROP VIEW IF EXISTS service_platforms_hourly_mv", + `CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage, + max(ResourceAttributes['browser.platform']) AS BrowserPlatform, + max(ResourceAttributes['device.type']) AS DeviceType, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv`, + ], + requiredForIngest: false, +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 5bfbe9183..7cbd3579e 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -17,6 +17,7 @@ import { migration_0011_session_analytics_columns } from "./0011_session_analyti import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events, webEventsBackfill } from "./0014_web_events" +import { migration_0015_service_app_kind } from "./0015_service_app_kind" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -31,15 +32,44 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) - expect(migrations.at(-1)).toBe(migration_0014_web_events) - expect(latestMigrationVersion).toBe(14) - // 0010 and 0014 are performance-only, so the ingest-gating version skips - // both and stays at 13 — nothing writes `web_events` directly, and bumping - // it would un-ready every BYO-CH org's ingest routing for a read-path change. + expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + expect(migrations.at(-1)).toBe(migration_0015_service_app_kind) + expect(latestMigrationVersion).toBe(15) + // 0010, 0014 and 0015 are read-path-only, so the ingest-gating version skips + // all three and stays at 13 — nothing writes `web_events` or + // `service_platforms_hourly` directly, and bumping it would un-ready every + // BYO-CH org's ingest routing for a read-path change. expect(clickHouseSchemaVersion).toBe("13") expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) + expect(migration_0015_service_app_kind.requiredForIngest).toBe(false) + }) + + it("appends the app-kind signal columns without rewriting service_platforms_hourly", () => { + const sql = migration_0015_service_app_kind.statements.filter((stmt) => !isBackfill(stmt)).join("\n") + + // The vendor-neutral markers: `maple.sdk.type` alone only ever classifies + // services instrumented with a Maple SDK. + expect(sql).toContain( + "ALTER TABLE service_platforms_hourly ADD COLUMN IF NOT EXISTS TelemetrySdkLanguage", + ) + expect(sql).toContain("max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage") + expect(sql).toContain("max(ResourceAttributes['browser.platform']) AS BrowserPlatform") + expect(sql).toContain("max(ResourceAttributes['device.type']) AS DeviceType") + + // The table already exists, so the view has to be swapped for the columns + // to ever be written — but nothing existing is dropped or rewritten. + expect(sql).toContain("DROP VIEW IF EXISTS service_platforms_hourly_mv") + expect(sql).toContain( + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly", + ) + expect(sql).not.toContain("DROP TABLE") + expect(sql).not.toContain("POPULATE") + + // No backfill: `max()` over the viewed window means one hour of fresh + // telemetry classifies the service, and the only `sum` column on the table + // (SpanCount) would double-count if re-inserted. + expect(migration_0015_service_app_kind.statements.some(isBackfill)).toBe(false) }) it("installs web_events with a live-write MV and no POPULATE", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index b719e9d98..9e62cedfe 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -13,6 +13,7 @@ import { migration_0011_session_analytics_columns } from "./0011_session_analyti import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events } from "./0014_web_events" +import { migration_0015_service_app_kind } from "./0015_service_app_kind" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -58,6 +59,7 @@ export const migrations: ReadonlyArray = [ migration_0012_session_event_attribute_keys, migration_0013_service_map_ingest_bridge, migration_0014_web_events, + migration_0015_service_app_kind, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 8db9cd4ff..9a377596e 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" as const +export const projectRevision = "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -30,7 +30,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS service_operations_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n SpanName String,\n SpanCount SimpleAggregateFunction(sum, UInt64),\n EstimatedSpanCount SimpleAggregateFunction(sum, Float64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationSum SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Minute)\nORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName)\nTTL toDate(Minute) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS service_overview_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ServiceNamespace LowCardinality(String),\n CommitSha LowCardinality(String),\n SpanCount SimpleAggregateFunction(sum, UInt64),\n EstimatedSpanCount SimpleAggregateFunction(sum, Float64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationSum SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64),\n ApdexToleratingCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Hour)\nORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_overview_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", - "CREATE TABLE IF NOT EXISTS service_platforms_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS service_platforms_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n TelemetrySdkLanguage SimpleAggregateFunction(max, String),\n BrowserPlatform SimpleAggregateFunction(max, String),\n DeviceType SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_usage (\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (OrgId, ServiceName, Hour)\nTTL Hour + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS session_events (\n OrgId LowCardinality(String),\n SessionId String,\n Timestamp DateTime64(9),\n Seq UInt32 DEFAULT 0,\n Type LowCardinality(String),\n Url String DEFAULT '',\n TraceId String DEFAULT '',\n Level LowCardinality(String) DEFAULT '',\n Message String DEFAULT '',\n TargetSelector String DEFAULT '',\n TargetText String DEFAULT '',\n NetMethod LowCardinality(String) DEFAULT '',\n NetUrl String DEFAULT '',\n NetStatus UInt16 DEFAULT 0,\n NetDurationMs UInt32 DEFAULT 0,\n ErrorStack String DEFAULT '',\n Attributes Map(String, String),\n INDEX idx_type Type TYPE set(16) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, Timestamp, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replay_events (\n OrgId LowCardinality(String),\n SessionId String,\n ChunkSeq UInt32,\n Timestamp DateTime64(9),\n DurationMs UInt32 DEFAULT 0,\n EventCount UInt32 DEFAULT 0,\n ByteSize UInt32 DEFAULT 0,\n Events String,\n IsCheckpoint UInt8 DEFAULT 0\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, ChunkSeq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -64,7 +64,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(toDateTime(Timestamp)) AS Minute,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles\n FROM traces\n GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['service.namespace'] AS ServiceNamespace,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n count() AS SpanCount,\n sum(SampleRate) AS EstimatedSpanCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration)) AS DurationSum,\n quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles,\n min(toDateTime(Timestamp)) AS FirstSeen,\n countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount,\n countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage,\n max(ResourceAttributes['browser.platform']) AS BrowserPlatform,\n max(ResourceAttributes['device.type']) AS DeviceType,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS\nSELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 82957e5e2..0eefa0b95 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" as const +export const projectRevision = "7637e9d59858fd4d8b7c019e1be7feac249120a6728d7ad4314de1276531677a" as const export const datasources = [ { @@ -137,7 +137,7 @@ export const datasources = [ { name: "service_platforms_hourly", content: - 'DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) for the service map\'s hosting-icon resolver. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 365 DAY"', + 'DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) plus app-kind signals (telemetry.sdk.language, browser.platform, device.type) for the service map\'s hosting-icon resolver and the service-detail app-kind classifier. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n TelemetrySdkLanguage SimpleAggregateFunction(max, String),\n BrowserPlatform SimpleAggregateFunction(max, String),\n DeviceType SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 365 DAY"', }, { name: "service_usage", @@ -310,7 +310,7 @@ export const pipes = [ { name: "service_platforms_hourly_mv", content: - "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", + "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) and app-kind signals (telemetry.sdk.language, browser.platform, device.type) into hourly buckets for the service map's runtime-icon resolver and the app-kind classifier.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n max(ResourceAttributes['telemetry.sdk.language']) AS TelemetrySdkLanguage,\n max(ResourceAttributes['browser.platform']) AS BrowserPlatform,\n max(ResourceAttributes['device.type']) AS DeviceType,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", }, { name: "service_usage_logs_mv", diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 9018341f9..313b08149 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -18,6 +18,7 @@ import { QueryEngineExecuteResponse, TinybirdDateTime, } from "../query-engine" +import { ServiceAppKind } from "../service-app-kind" import { Authorization } from "./current-tenant" import { warehouseHttpErrors } from "./warehouse" @@ -654,6 +655,28 @@ export class ServiceDetailOverviewResponse extends Schema.Class( @@ -747,7 +770,10 @@ export class ServicePlatformsRequest extends Schema.Class): ServiceAppKindSignals => ({ + ...NO_SIGNALS, + ...overrides, +}) + +describe("classifyServiceAppKind", () => { + const cases: ReadonlyArray = [ + // Maple's own browser SDK writes "browser" (packages/browser/src/tracing.ts). + // It classified as `unknown` before this signal existed, which is exactly how + // every browser app ended up scored against a 500 ms backend target. + ["maple browser SDK", signals({ mapleSdkType: "browser" }), "browser"], + ["maple effect client SDK", signals({ mapleSdkType: "client" }), "browser"], + // The case the vendor-neutral signals exist for: no Maple SDK anywhere. + ["vanilla OTel web", signals({ telemetrySdkLanguage: "webjs" }), "browser"], + ["browser.platform alone", signals({ browserPlatform: "macOS" }), "browser"], + ["maple mobile SDK", signals({ mapleSdkType: "mobile" }), "mobile"], + ["swift SDK", signals({ telemetrySdkLanguage: "swift" }), "mobile"], + ["device.type alone", signals({ deviceType: "phone" }), "mobile"], + ["kubernetes pod", signals({ k8sPodName: "api-7d9f-x2k" }), "backend"], + [ + "cloudflare worker", + signals({ cloudPlatform: "cloudflare.workers", cloudProvider: "cloudflare" }), + "backend", + ], + ["lambda", signals({ faasName: "checkout-handler" }), "backend"], + ["maple server SDK", signals({ mapleSdkType: "server" }), "backend"], + ["no signals at all", NO_SIGNALS, "unknown"], + ] + + for (const [name, input, expected] of cases) { + it(`classifies ${name} as ${expected}`, () => { + expect(classifyServiceAppKind(input)).toBe(expected) + }) + } + + // A browser app can pick up `cloud.provider` from a CDN or a `k8s.*` leak from + // an OTel gateway it was proxied through. Neither makes it a backend, and the + // reverse mistake is impossible — a server never reports `browser.platform`. + it("prefers browser over host-infrastructure signals on the same service", () => { + expect( + classifyServiceAppKind( + signals({ + browserPlatform: "Windows", + cloudProvider: "cloudflare", + k8sPodName: "otel-gateway-abc", + }), + ), + ).toBe("browser") + }) + + it("prefers mobile over host-infrastructure signals on the same service", () => { + expect(classifyServiceAppKind(signals({ mapleSdkType: "mobile", cloudProvider: "aws" }))).toBe( + "mobile", + ) + }) +}) + +describe("apdexThresholdMsForAppKind", () => { + it("keeps the backend default for backend and unknown", () => { + expect(apdexThresholdMsForAppKind("backend")).toBe(DEFAULT_APDEX_THRESHOLD_MS) + expect(apdexThresholdMsForAppKind("unknown")).toBe(DEFAULT_APDEX_THRESHOLD_MS) + }) + + // 2500 ms is the Core Web Vitals "good" LCP boundary, which puts the + // frustrated line (4T, per the Apdex spec) at 10s. + it("scores a browser app against the Core Web Vitals boundary", () => { + expect(apdexThresholdMsForAppKind("browser")).toBe(2500) + }) + + it("raises the mobile target above backend but below browser", () => { + expect(APDEX_THRESHOLD_MS_BY_APP_KIND.mobile).toBeGreaterThan(DEFAULT_APDEX_THRESHOLD_MS) + expect(APDEX_THRESHOLD_MS_BY_APP_KIND.mobile).toBeLessThan(APDEX_THRESHOLD_MS_BY_APP_KIND.browser) + }) +}) diff --git a/packages/domain/src/service-app-kind.ts b/packages/domain/src/service-app-kind.ts new file mode 100644 index 000000000..be5a1a269 --- /dev/null +++ b/packages/domain/src/service-app-kind.ts @@ -0,0 +1,115 @@ +import { Schema } from "effect" + +/** + * What kind of application a service *is* — deliberately orthogonal to + * `ServicePlatform`, which says where it *runs*. A browser app has no hosting + * platform; a Kubernetes pod can be a backend or a batch worker. + * + * The distinction earns its keep in one place today: the Apdex threshold. Apdex + * scores a request against a satisfaction target T, and 500 ms is a target for a + * backend API. Applied to a browser app — where a span is a request made from a + * device on someone's home wifi — it scores every real-world user as frustrated + * and the chart stops carrying signal. + */ +export const ServiceAppKind = Schema.Literals(["browser", "mobile", "backend", "unknown"]) +export type ServiceAppKind = Schema.Schema.Type + +/** The Apdex target for a service whose kind could not be determined, and the + * value every caller that has no service in hand (dashboards, ad-hoc queries, + * alert rules) uses. Also the constant baked into + * `service_overview_hourly.ApdexSatisfiedCount` / `ApdexToleratingCount`, which + * is why the rollup path is only valid at exactly this threshold. */ +export const DEFAULT_APDEX_THRESHOLD_MS = 500 + +/** + * Apdex T per app kind. Frustrated starts at 4T in every case (the Apdex spec), + * so these also set the ceilings: 2 s / 10 s for a browser app, 4 s for mobile. + * + * `browser` is 2500 ms because that is the Core Web Vitals "good" LCP boundary — + * the number the rest of the industry already uses for "this page felt fast" — + * and it puts the frustrated line at 10 s. `mobile` sits between the two: a + * native app's network calls are slower than a datacenter's and faster than a + * cold page load. + */ +export const APDEX_THRESHOLD_MS_BY_APP_KIND: Readonly> = Object.freeze({ + browser: 2500, + mobile: 1000, + backend: DEFAULT_APDEX_THRESHOLD_MS, + unknown: DEFAULT_APDEX_THRESHOLD_MS, +}) + +export const apdexThresholdMsForAppKind = (kind: ServiceAppKind): number => + APDEX_THRESHOLD_MS_BY_APP_KIND[kind] + +/** + * The resource-attribute signals the classifier reads, as stored per hour in + * `service_platforms_hourly`. Every field is "" when the attribute was absent. + */ +export interface ServiceAppKindSignals { + /** `browser.platform` — per OTel semconv, only ever set in a browser. */ + readonly browserPlatform: string + /** `telemetry.sdk.language` — `webjs` is the OTel browser SDK. */ + readonly telemetrySdkLanguage: string + /** `maple.sdk.type` — set only by Maple's own SDKs. */ + readonly mapleSdkType: string + /** `device.type` — the mobile-side marker. */ + readonly deviceType: string + readonly cloudPlatform: string + readonly cloudProvider: string + readonly faasName: string + readonly k8sPodName: string + readonly k8sDeploymentName: string +} + +/** `telemetry.sdk.language` values that only a mobile SDK reports. */ +const MOBILE_SDK_LANGUAGES = new Set(["swift", "objc", "kotlin", "android"]) + +/** + * Classify a service from its resource attributes, first match wins. + * + * Browser is checked before everything else on purpose: a browser app can carry + * `cloud.provider` (a CDN-injected attribute) or a `k8s.*` leak from an OTel + * gateway it was proxied through, and neither makes it a backend. The reverse + * mistake is not possible — a server never reports `browser.platform`. + * + * All-empty signals return `unknown` rather than guessing `backend`: `unknown` + * and `backend` resolve to the same 500 ms threshold, so the honest answer costs + * nothing, and the UI can decline to render a badge it isn't sure about. + */ +export const classifyServiceAppKind = (signals: ServiceAppKindSignals): ServiceAppKind => { + if ( + signals.browserPlatform !== "" || + signals.telemetrySdkLanguage === "webjs" || + signals.mapleSdkType === "browser" || + signals.mapleSdkType === "client" + ) { + return "browser" + } + if ( + signals.mapleSdkType === "mobile" || + MOBILE_SDK_LANGUAGES.has(signals.telemetrySdkLanguage) || + signals.deviceType !== "" + ) { + return "mobile" + } + if ( + signals.k8sPodName !== "" || + signals.k8sDeploymentName !== "" || + signals.cloudPlatform !== "" || + signals.cloudProvider !== "" || + signals.faasName !== "" || + signals.mapleSdkType !== "" + ) { + return "backend" + } + return "unknown" +} + +/** Human label for the app-kind badge. `unknown` has none — the UI renders + * nothing rather than a badge that says it doesn't know. */ +export const SERVICE_APP_KIND_LABELS: Readonly, string>> = + Object.freeze({ + browser: "Browser", + mobile: "Mobile", + backend: "Backend", + }) diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 048fc4b7f..bae05f1cf 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -679,11 +679,18 @@ export type ServiceAddressResolutionsHourlyRow = InferRow { expect(sql).toContain("quantilesTDigestMerge(0.5, 0.95, 0.99)(bDurationQuantiles)") }) + // The service-detail Overview tab is exactly the request above, and it must + // stay that way. `service_overview_hourly.ApdexSatisfiedCount` is computed at + // a hardcoded 500 ms, so a kind-aware Apdex target cannot be threaded through + // this request — it would drop throughput, latency, AND error rate onto the + // 30-day raw path for the sake of one series. The handler re-scores Apdex with + // a second, narrower query instead; this asserts the fork it depends on. + it("drops the annual rollup when a non-default apdex threshold is requested", () => { + const base = { + metric: "count" as const, + needsSampling: true, + allMetrics: true, + rootOnly: true, + bucketSeconds: 3600, + serviceName: "api", + } + expect(canUseAnnualServiceOverview(base)).toBe(true) + expect(canUseAnnualServiceOverview({ ...base, apdexThresholdMs: 500 })).toBe(true) + // A browser service's 2500 ms target. + expect(canUseAnnualServiceOverview({ ...base, apdexThresholdMs: 2500 })).toBe(false) + + const { sql } = compileCH(tracesTimeseriesQuery({ ...base, apdexThresholdMs: 2500 }), baseParams) + expect(sql).not.toContain("FROM service_overview_hourly") + }) + // `service_overview_spans` stores only entry-point spans (Server/Consumer OR // root). Routing an all-spans query there silently swaps the population, which // is how one dashboard showed two answers to the same question. diff --git a/packages/query-engine/src/ch/queries/service-map.test.ts b/packages/query-engine/src/ch/queries/service-map.test.ts index c272b486b..16b1a20e6 100644 --- a/packages/query-engine/src/ch/queries/service-map.test.ts +++ b/packages/query-engine/src/ch/queries/service-map.test.ts @@ -824,6 +824,9 @@ describe("servicePlatformsSQL", () => { faasName: "", mapleSdkType: "node", processRuntimeName: "nodejs", + telemetrySdkLanguage: "nodejs", + browserPlatform: "", + deviceType: "", }, ]) @@ -832,6 +835,8 @@ describe("servicePlatformsSQL", () => { k8sDeploymentName: "artifacts-api", cloudProvider: "aws", mapleSdkType: "node", + telemetrySdkLanguage: "nodejs", + browserPlatform: "", }) }), ) @@ -858,4 +863,22 @@ describe("servicePlatformsSQL", () => { expect(Exit.isFailure(exit)).toBe(true) }), ) + + it("selects the app-kind signal columns", () => { + const sql = servicePlatformsSQL({}, baseParams).sql + + // `maple.sdk.type` alone only classifies services on a Maple SDK; these + // three are what let the classifier see a vanilla-OTel browser or mobile app. + expect(sql).toContain("TelemetrySdkLanguage") + expect(sql).toContain("BrowserPlatform") + expect(sql).toContain("DeviceType") + }) + + it("narrows to one service when asked", () => { + const all = servicePlatformsSQL({}, baseParams).sql + const one = servicePlatformsSQL({ serviceName: "artifacts-api" }, baseParams).sql + + expect(all).not.toContain("artifacts-api") + expect(one).toContain("artifacts-api") + }) }) diff --git a/packages/query-engine/src/ch/queries/service-map.ts b/packages/query-engine/src/ch/queries/service-map.ts index 3d8cffcd6..051af5746 100644 --- a/packages/query-engine/src/ch/queries/service-map.ts +++ b/packages/query-engine/src/ch/queries/service-map.ts @@ -1189,6 +1189,8 @@ export function serviceExternalEdgesSQL( export interface ServicePlatformsOpts { deploymentEnv?: string + /** Narrow to one service — the service-detail app-kind lookup. */ + serviceName?: string } export interface ServicePlatformsOutput { @@ -1201,6 +1203,9 @@ export interface ServicePlatformsOutput { readonly faasName: string readonly mapleSdkType: string readonly processRuntimeName: string + readonly telemetrySdkLanguage: string + readonly browserPlatform: string + readonly deviceType: string } const ServicePlatformsOutputSchema: CompiledQueryRowSchema = Schema.Struct({ @@ -1213,6 +1218,9 @@ const ServicePlatformsOutputSchema: CompiledQueryRowSchema [ $.OrgId.eq(param.string("orgId")), $.Hour.gte(CH.toStartOfHour(CH.toDateTime(param.dateTime("startTime")))), $.Hour.lte(param.dateTime("endTime")), $.ServiceName.neq(""), + opts.serviceName ? $.ServiceName.eq(opts.serviceName) : undefined, opts.deploymentEnv ? $.DeploymentEnv.eq(opts.deploymentEnv) : undefined, ]) .groupBy("serviceName") diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 998ce8763..38bf28c30 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -480,6 +480,9 @@ export const ServicePlatformsHourly = table("service_platforms_hourly", { FaasName: T.string, MapleSdkType: T.string, ProcessRuntimeName: T.string, + TelemetrySdkLanguage: T.string, + BrowserPlatform: T.string, + DeviceType: T.string, SpanCount: T.uint64, }) diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index 0f956bc40..b1f5f8b71 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -543,6 +543,29 @@ export const servicePlatforms = defineQuery({ ), }) +/** + * The same platform/app-kind row for a single service — the service-detail + * lookup that picks the page's Apdex threshold. + * + * Cached for 5 minutes rather than the usual 15 seconds: a service's app kind + * is a property of how it is instrumented, so it changes on deploy at most, and + * this read sits in front of the Overview tab's timeseries. On a cold key it + * costs one small aggregate-table scan; on every other load it costs nothing. + */ +export const serviceAppKind = defineQuery({ + id: "serviceAppKind", + profile: "aggregation", + cache: 300, + compile: ( + payload: { serviceName: string; startTime: string; endTime: string; deploymentEnv?: string }, + orgId: string, + ) => + CH.servicePlatformsSQL( + { serviceName: payload.serviceName, deploymentEnv: payload.deploymentEnv }, + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + const dbQueryParams = (payload: ServiceDbQuerySummaryRequest, orgId: string) => ({ orgId, dbSystem: payload.dbSystem, diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index a084f9030..328c60c6a 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -16,6 +16,9 @@ import { LOCAL_SCHEMA_V4, LOCAL_SCHEMA_V4_MANIFEST_DIGEST, LOCAL_SCHEMA_V4_SQL, + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST_DIGEST, + LOCAL_SCHEMA_V5_SQL, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -104,6 +107,18 @@ if ( fail("the immutable local schema v4 snapshot no longer matches its historical identity") } +const v5 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V5.version) +if ( + !v5 || + LOCAL_SCHEMA_V5_MANIFEST_DIGEST !== v5.manifestDigest || + schemaFingerprint(LOCAL_SCHEMA_V5_SQL) !== v5.fingerprint || + schemaDigest(LOCAL_SCHEMA_V5_SQL) !== v5.digest || + LOCAL_SCHEMA_V5.fingerprint !== v5.fingerprint || + LOCAL_SCHEMA_V5.digest !== v5.digest +) { + fail("the immutable local schema v5 snapshot no longer matches its historical identity") +} + const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name) if (new Set(names).size !== names.length) fail("local structural schema manifest contains duplicate object names")