From 19aa5a13b87bd75ebafbdc50c3a1547a75806d79 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 11 Aug 2026 22:26:27 +0200 Subject: [PATCH] feat(ingest): classify AI spans on the write path and store the columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds five defaulted trailing columns to `traces` (`AiVendor`, `AiSessionKeyState`, `AiSessionKeyHash`, `AiRulesVersion`, `AiRollupHour`) plus two skip indexes, as ClickHouse migration 0015 and local-store migration v4 -> v5, and wires the classifier from level 3 into the Rust gateway's row writer, the local-mode OTLP encoder and the ingest metrics. The columns are all DEFAULT-valued so the ALTER is metadata-only — no mutation on a table whose 30-day TTL retires unindexed parts on its own — and rows written before the classifier shipped stay readable, with `AiRulesVersion = 0` meaning "never examined" as distinct from "examined, not AI". `AiRollupHour` is receive-time-clamped and written unconditionally so a skewed client cannot open a partition in 2038. **Operational note:** 0015 is `requiredForIngest: true`, so `clickHouseSchemaVersion` moves from 13 to 15. The gateway's INSERT now names all five AI columns, and a BYO cluster without them would reject every direct insert — so BYO-ClickHouse orgs resolve `clickhouse_ready = false` and route to the managed pipeline until their schema syncs 0015. That fallback is the designed behaviour, not a regression, but it is the reason this level should not ship without the schema rollout. Co-Authored-By: Claude Fable 5 --- ...rehouseQueryService.clickhouse.e2e.test.ts | 86 + apps/cli/src/server/local-schema-history.ts | 7 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 3 + .../v4-to-v5-ai-classification-columns.ts | 257 +++ apps/cli/src/server/otlp/encode.ts | 54 + apps/cli/src/server/schema-identity.ts | 17 +- apps/cli/src/server/schema/local-inserts.json | 18 +- .../cli/src/server/schema/local-schema-v5.sql | 1703 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 13 +- apps/cli/test/local-store-migrations.test.ts | 51 +- apps/ingest/benches/ingest_bench.rs | 6 +- apps/ingest/src/clickhouse_insert_mappings.rs | 10 +- apps/ingest/src/main.rs | 239 ++- apps/ingest/src/metrics.rs | 19 + apps/ingest/src/otel.rs | 6 + apps/ingest/src/telemetry.rs | 598 +++++- .../0015_ai_classification_columns.ts | 60 + .../src/clickhouse/migrations/index.test.ts | 49 +- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 4 +- .../generated/tinybird-project-manifest.ts | 4 +- .../src/tinybird/datasources.contract.test.ts | 20 +- packages/domain/src/tinybird/datasources.ts | 47 + scripts/check-local-schema-manifest.ts | 15 + 25 files changed, 3215 insertions(+), 75 deletions(-) create mode 100644 apps/cli/src/server/local-store-migrations/v4-to-v5-ai-classification-columns.ts create mode 100644 apps/cli/src/server/schema/local-schema-v5.sql create mode 100644 packages/domain/src/clickhouse/migrations/0015_ai_classification_columns.ts diff --git a/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts index 84ddec2c1..b0e1bd17b 100644 --- a/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts @@ -22,6 +22,9 @@ import { const enabled = clickhouseE2eEnabled const database = uniqueDatabase("maple_raw_sql_e2e") const orgId = "org_raw_sql_e2e" +/** Isolates the migration-0015 default-readback probe row from the row-level + * fixtures the query tests assert on. */ +const aiProbeOrgId = "org_raw_sql_e2e_ai_probe" const assertSearchSchemaApplied = async (): Promise => { const migrationRevision = ( @@ -140,6 +143,88 @@ SETTINGS enable_full_text_index = 1`, assert.include(explain, "idx_lower_body_text") } +/** + * Migration 0015 is storage-only — nothing writes these columns yet — so the + * only thing that can prove it applied is the physical schema. Asserted against + * a real server because `set(0)` and `tokenbf_v1` are the kind of DDL a + * SQL-text test happily accepts and ClickHouse rejects. + */ +const assertAiClassificationSchemaApplied = async (): Promise => { + const migrationRevision = ( + await clickhouseExec( + "SELECT count() FROM _maple_schema_migrations WHERE version = 15 FORMAT TabSeparated", + database, + ) + ).trim() + assert.strictEqual(migrationRevision, "1", "AI classification migration 15 was not recorded") + + const columns = ( + await clickhouseExec( + `SELECT name, type, default_kind +FROM system.columns +WHERE database = currentDatabase() + AND table = 'traces' + AND name IN ('AiVendor', 'AiSessionKeyState', 'AiSessionKeyHash', 'AiRulesVersion', 'AiRollupHour') +ORDER BY name +FORMAT TabSeparated`, + database, + ) + ) + .trim() + // TabSeparated escapes single quotes, so `DateTime('UTC')` arrives as + // `DateTime(\'UTC\')`. Unescape rather than encode the escaping into the + // expectation, which would read as a typo. + .replaceAll("\\'", "'") + assert.strictEqual( + columns, + [ + "AiRollupHour\tDateTime('UTC')\tDEFAULT", + "AiRulesVersion\tUInt32\tDEFAULT", + "AiSessionKeyHash\tUInt64\tDEFAULT", + "AiSessionKeyState\tUInt8\tDEFAULT", + "AiVendor\tLowCardinality(String)\tDEFAULT", + ].join("\n"), + "AI classification columns are missing, mistyped, or not default-computed", + ) + + const indexes = ( + await clickhouseExec( + `SELECT name, type_full, granularity +FROM system.data_skipping_indices +WHERE database = currentDatabase() + AND table = 'traces' + AND name IN ('idx_ai_vendor', 'idx_scope_name') +ORDER BY name +FORMAT TabSeparated`, + database, + ) + ).trim() + assert.strictEqual( + indexes, + ["idx_ai_vendor\tset(0)\t4", "idx_scope_name\ttokenbf_v1(4096, 3, 0)\t4"].join("\n"), + "AI classification skip indexes are missing or declared with the wrong type", + ) + + // A writer that names none of the new columns — which is every writer until + // the classifier ships — must still land a readable row. Written under its own + // org so it stays out of the row-level fixtures the tests below assert on. + await clickhouseExec( + `INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode) + VALUES ('${aiProbeOrgId}', now64(9), 'trace-ai-default', 'span-ai-default', 'GET /ai', 'Server', 'api', 1, 'Ok')`, + database, + ) + const defaults = ( + await clickhouseExec( + `SELECT AiVendor = '', AiSessionKeyState, AiSessionKeyHash, AiRulesVersion, toUnixTimestamp(AiRollupHour) +FROM traces +WHERE OrgId = '${aiProbeOrgId}' AND TraceId = 'trace-ai-default' +FORMAT TabSeparated`, + database, + ) + ).trim() + assert.strictEqual(defaults, "1\t0\t0\t0\t0", "AI classification defaults do not read back") +} + const trackedDbs: TestDb[] = [] const asOrgId = Schema.decodeUnknownSync(OrgId) const asUserId = Schema.decodeUnknownSync(UserId) @@ -196,6 +281,7 @@ describe.skipIf(!enabled)("WarehouseQueryService ClickHouse raw-SQL E2E", () => await clickhouseExec(`CREATE DATABASE ${database}`) await applyRealMigrations(database) await assertSearchSchemaApplied() + await assertAiClassificationSchemaApplied() await clickhouseExec( `INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index c5d051f39..48c77f2ad 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -51,4 +51,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "826f9363db5dd7722debd0c87a5b74a5b66387f4752abc219d4cc0ce76358a9e", projectRevision: "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a", }), + Object.freeze({ + version: 5, + fingerprint: "b3059dd34e85858f", + digest: "b3059dd34e85858f8893cd7fc88d9c28f489992c39fb2a334f8caf1747a69c21", + manifestDigest: "e0b0e0a9af30cc7aca51cec02c566dab9f4cbfda1374c177a7caee9a46a31783", + projectRevision: "09513d18e8cdea657efa56dbe764defebe66a28e5397411dc03fadb7f19f1c58", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 19b611ed5..40530abf3 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 4 as const +export const LOCAL_SCHEMA_VERSION = 5 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index de0080783..cecf0af6e 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -36,6 +36,7 @@ import { legacyToCurrentModule } from "./local-store-migrations/legacy-to-curren import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error-rollup" import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" +import { v4ToV5AiClassificationColumnsModule } from "./local-store-migrations/v4-to-v5-ai-classification-columns" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -58,6 +59,7 @@ export { legacyToCurrentModule } from "./local-store-migrations/legacy-to-curren export { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error-rollup" export { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" export { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" +export { v4ToV5AiClassificationColumnsModule } from "./local-store-migrations/v4-to-v5-ai-classification-columns" const NONTERMINAL_PHASES = new Set([ "planned", @@ -122,6 +124,7 @@ export const localStoreMigrations: ReadonlyArray = v1ToV2ErrorRollupModule, v2ToV3ServiceMapIngestBridgeModule, v3ToV4WebEventsModule, + v4ToV5AiClassificationColumnsModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v4-to-v5-ai-classification-columns.ts b/apps/cli/src/server/local-store-migrations/v4-to-v5-ai-classification-columns.ts new file mode 100644 index 000000000..32dc4d88e --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v4-to-v5-ai-classification-columns.ts @@ -0,0 +1,257 @@ +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V4, + LOCAL_SCHEMA_V4_MANIFEST, + LOCAL_SCHEMA_V4_SQL, + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V5_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +/** + * Byte-for-byte the structural half of ClickHouse migration 0015. `traces` + * already exists here, so bootstrapping the v5 DDL is a no-op on it + * (`CREATE TABLE IF NOT EXISTS`) — the columns and indexes only arrive through + * these ALTERs. If 0015 changes, change these together or a migrated local + * store stops matching the bundled v5 manifest, which `verify` below catches. + */ +const AI_CLASSIFICATION_STATEMENTS: ReadonlyArray = [ + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiVendor LowCardinality(String) DEFAULT ''", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiSessionKeyState UInt8 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiSessionKeyHash UInt64 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiRulesVersion UInt32 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiRollupHour DateTime('UTC') DEFAULT toDateTime(0)", + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4", + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4", +] + +interface V4ToV5State { + readonly module: "local-0004-to-0005-ai-classification-columns" + readonly version: 1 + readonly rawRows: Readonly> + readonly retentionDays?: number +} + +interface V4ToV5Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v4 -> v5 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (typeof count !== "string" || !/^\d+$/.test(count)) + throw new Error(`v4 -> v5 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v4 -> v5 rawRows contains an unknown table") + return counts +} + +const decodeState = (value: unknown): V4ToV5State => { + if (!isRecord(value)) throw new Error("v4 -> v5 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v4 -> v5 state contains an unknown field") + if (value.module !== "local-0004-to-0005-ai-classification-columns" || value.version !== 1) + throw new Error("v4 -> v5 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v4 -> v5 retentionDays must be an integer") + return { + module: "local-0004-to-0005-ai-classification-columns", + version: 1, + rawRows: decodeCounts(value.rawRows), + ...(value.retentionDays === undefined ? {} : { retentionDays: value.retentionDays }), + } +} + +const decodeProgress = (value: unknown): V4ToV5Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v4 -> v5 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V4_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V4_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V4_SQL, bootstrapSchema: false }, + ) + return { + module: "local-0004-to-0005-ai-classification-columns", + version: 1, + rawRows, + ...(retentionDays === undefined ? {} : { retentionDays }), + } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V4ToV5State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Metadata-only and additive: five trailing columns with constant DEFAULTs plus + * two skip indexes on `traces`. No part is rewritten — existing parts read the + * defaults for free, and every ALTER is `IF NOT EXISTS`, so a resumed run + * converges instead of failing. + * + * Deliberately no `MATERIALIZE INDEX`: that is a mutation over the whole table, + * and the store's 30-day raw-telemetry TTL rolls every unindexed part out on its + * own. Same reasoning as ClickHouse migration 0015 — the local store is not a + * different decision, just a smaller one. + * + * Nothing writes the new columns yet (the ingest classifier lands in a later + * stage), so the migrated store's `traces` rows all carry the defaults: AiVendor + * '' = not classified, AiRulesVersion 0 = the row predates classification. + */ +const apply = async (context: MigrationModuleContext): Promise => + context.openTarget( + (db) => { + for (const statement of AI_CLASSIFICATION_STATEMENTS) db.exec(statement) + return { installed: true } as const + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: true }, + ) + +const verify = async ( + context: MigrationModuleContext, + state: V4ToV5State, + _progress: V4ToV5Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V5_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v4 -> v5 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v4-store", + description: "Clone the stopped v4 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "add-ai-classification-columns", + description: "Add the AI classification columns and vendor/scope skip indexes to traces", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v5-schema", + description: "Verify the v5 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v4 store is cloned byte-for-byte before additive DDL runs.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "Trailing defaulted columns and skip indexes are metadata-only; no existing row or part is rewritten and every prior column keeps its value.", + }, + { + // The new columns are readable immediately — every pre-migration row reads + // its DEFAULT — but they are not *classified*: nothing writes them until the + // ingest classifier ships, and AiRulesVersion = 0 is precisely the marker + // for "this row was never examined". The skip indexes cover only parts + // written after the ALTER; the 30-day TTL retires the rest. + name: "traces AI classification columns", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Defaulted for every existing row (AiRulesVersion 0 = never classified) and filled forward once the ingest classifier ships; complete by construction after the 30-day raw-telemetry horizon.", + preservationInterval: "traces retention horizon", + sourceRetentionDays: 30, + targetRetentionDays: 30, + }, +] + +export const v4ToV5AiClassificationColumnsModule: LocalStoreMigrationModule = { + id: "local-0004-to-0005-ai-classification-columns", + moduleVersion: 1, + description: "Add the AI classification columns and vendor/scope skip indexes to traces on v4", + from: LOCAL_SCHEMA_V4, + to: LOCAL_SCHEMA_V5, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/otlp/encode.ts b/apps/cli/src/server/otlp/encode.ts index 2aad0d69a..36ed4a5fe 100644 --- a/apps/cli/src/server/otlp/encode.ts +++ b/apps/cli/src/server/otlp/encode.ts @@ -200,6 +200,46 @@ export function formatTimestampNano(nanos: string | number | undefined): string return `${calendar}.${frac9}` } +/** Seconds in the clamp window's past and future halves (write-side plan §3). */ +const ROLLUP_CLAMP_PAST_SECS = 7 * 24 * 60 * 60 +const ROLLUP_CLAMP_FUTURE_SECS = 24 * 60 * 60 + +/** + * Port of Rust `rollup_hour_secs` + `format_datetime_secs` (see + * `apps/ingest/src/telemetry.rs`). `AiRollupHour` is `toStartOfHour(start time)` + * when the span's start is within `[receive − 7d, receive + 1d]`, else + * `toStartOfHour(receive time)`, rendered as `DateTime('UTC')`'s + * `YYYY-MM-DD HH:MM:SS`. + * + * Local mode has no classifier, so the other four AI columns stay at their + * "never examined" defaults — but this one is written for real on every span, + * exactly as the gateway does. It is `service_ai_vendors_hourly`'s partition + * key, so an unclamped client timestamp means unbounded partition creation and + * rows whose TTL never fires. Clamping at write time is also what keeps a later + * partition rebuild from relocating rows: an MV-side clamp would need `now()`, + * re-evaluated at rebuild time. + */ +export function formatRollupHour( + spanStartUnixNano: string | number | undefined, + receiveTimeSecs: number, +): string { + let spanSecs = 0 + try { + spanSecs = Number(BigInt(spanStartUnixNano ?? 0) / 1_000_000_000n) + } catch { + spanSecs = 0 + } + const inWindow = + Number.isFinite(spanSecs) && + spanSecs >= receiveTimeSecs - ROLLUP_CLAMP_PAST_SECS && + spanSecs <= receiveTimeSecs + ROLLUP_CLAMP_FUTURE_SECS + const chosen = inWindow ? spanSecs : receiveTimeSecs + const hour = chosen - (((chosen % 3600) + 3600) % 3600) + const date = new Date(hour * 1000) + if (Number.isNaN(date.getTime())) return "1970-01-01 00:00:00" + return date.toISOString().slice(0, 19).replace("T", " ") +} + /** * Port of Rust `any_value_string`: coerce an OTLP `AnyValue` to a string * exactly as the Rust encoder does. @@ -540,6 +580,9 @@ interface Span { export function encodeTraces(req: unknown): EncodedBatch[] { const request = (req ?? {}) as TraceRequest const rows: Record[] = [] + // One receive time for the whole batch, matching the gateway: two spans in + // the same request must not be able to land on different clamp anchors. + const receiveTimeSecs = Math.floor(Date.now() / 1000) for (const resourceSpans of request.resourceSpans ?? []) { const resourceAttrs = attrMap(resourceSpans.resource?.attributes) @@ -586,6 +629,17 @@ export function encodeTraces(req: unknown): EncodedBatch[] { links_span_id: links.map((link, i) => spanIdHex(link.spanId, `span.links[${i}].spanId`)), links_trace_state: links.map((link) => link.traceState ?? ""), links_attributes: links.map((link) => attrMap(link.attributes)), + // Local mode runs no classifier, so these four stay at the + // "never examined" defaults the ClickHouse columns declare: + // AiVendor '' = not classified as AI, AiRulesVersion 0 = never + // looked at. `service_ai_vendors_hourly` filters on AiVendor != '', + // so a local store's rollup stays empty rather than reporting a + // confident zero — which is the honest answer here. + ai_vendor: "", + ai_session_key_state: 0, + ai_session_key_hash: 0, + ai_rules_version: 0, + ai_rollup_hour: formatRollupHour(span.startTimeUnixNano, receiveTimeSecs), }) } } diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index a3ba0c12a..4cfe6f033 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -3,6 +3,7 @@ import schemaV1Sql from "./schema/local-schema-v1.sql" with { type: "text" } import schemaV2Sql from "./schema/local-schema-v2.sql" with { type: "text" } import schemaV3Sql from "./schema/local-schema-v3.sql" with { type: "text" } import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } +import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -26,7 +27,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" + "09513d18e8cdea657efa56dbe764defebe66a28e5397411dc03fadb7f19f1c58" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -57,6 +58,11 @@ export const LOCAL_SCHEMA_V3_MANIFEST_DIGEST = LOCAL_SCHEMA_V3_MANIFEST.digest export const LOCAL_SCHEMA_V4_SQL = schemaV4Sql export const LOCAL_SCHEMA_V4_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV4Sql) export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest +/** Immutable v5 DDL/manifest snapshot used by the v4 -> v5 module after the + * generated current schema advances. */ +export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql +export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) +export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -107,6 +113,15 @@ export const LOCAL_SCHEMA_V4: LocalSchemaIdentity = Object.freeze({ projectRevision: LOCAL_SCHEMA_HISTORY[4]!.projectRevision, }) +export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_HISTORY[5]!.version, + fingerprint: LOCAL_SCHEMA_HISTORY[5]!.fingerprint, + digest: LOCAL_SCHEMA_HISTORY[5]!.digest, + manifestDigest: LOCAL_SCHEMA_HISTORY[5]!.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, +}) + export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, fingerprint: SCHEMA_FINGERPRINT, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index cebc793d3..4244958b6 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a", + "projectRevision": "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c", "orgPlaceholder": "__ORG__", "datasources": { "traces": { @@ -30,7 +30,12 @@ "LinksTraceId", "LinksSpanId", "LinksTraceState", - "LinksAttributes" + "LinksAttributes", + "AiVendor", + "AiSessionKeyState", + "AiSessionKeyHash", + "AiRulesVersion", + "AiRollupHour" ], "selects": [ "__ORG__", @@ -58,9 +63,14 @@ "links_trace_id", "links_span_id", "links_trace_state", - "links_attributes" + "links_attributes", + "ai_vendor", + "ai_session_key_state", + "ai_session_key_hash", + "ai_rules_version", + "ai_rollup_hour" ], - "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))" + "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String)), ai_vendor LowCardinality(String), ai_session_key_state UInt8, ai_session_key_hash UInt64, ai_rules_version UInt32, ai_rollup_hour DateTime('UTC')" }, "logs": { "table": "logs", diff --git a/apps/cli/src/server/schema/local-schema-v5.sql b/apps/cli/src/server/schema/local-schema-v5.sql new file mode 100644 index 000000000..67f5ae60d --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v5.sql @@ -0,0 +1,1703 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 09513d18e8cdea657efa56dbe764defebe66a28e5397411dc03fadb7f19f1c58 +-- localSchemaVersion: 5 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + AiVendor LowCardinality(String) DEFAULT '', + AiSessionKeyState UInt8 DEFAULT 0, + AiSessionKeyHash UInt64 DEFAULT 0, + AiRulesVersion UInt32 DEFAULT 0, + AiRollupHour DateTime('UTC') DEFAULT toDateTime(0), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4, + INDEX idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 59ffa044f..9c147d9ad 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a --- localSchemaVersion: 4 +-- projectRevision: 062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c +-- localSchemaVersion: 5 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -777,13 +777,20 @@ CREATE TABLE IF NOT EXISTS traces ( ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + AiVendor LowCardinality(String) DEFAULT '', + AiSessionKeyState UInt8 DEFAULT 0, + AiSessionKeyHash UInt64 DEFAULT 0, + AiRulesVersion UInt32 DEFAULT 0, + AiRollupHour DateTime('UTC') DEFAULT toDateTime(0), INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4, + INDEX idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 2df59982e..16c07b3e8 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -12,6 +12,9 @@ import { LOCAL_SCHEMA_V3, LOCAL_SCHEMA_V3_MANIFEST, LOCAL_SCHEMA_V4, + LOCAL_SCHEMA_V4_MANIFEST, + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -53,16 +56,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v4 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("75ac856927d88d56") - expect(SCHEMA_DIGEST).toBe("75ac856927d88d56518f12c68407a8f2a199d000b6eeb8576f9c97000138f5a4") + it("matches the generated v5 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("b3059dd34e85858f") + expect(SCHEMA_DIGEST).toBe("b3059dd34e85858f8893cd7fc88d9c28f489992c39fb2a334f8caf1747a69c21") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(4) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V4) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(5) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V5) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -97,8 +100,36 @@ describe("current local schema identity", () => { const v3Names = new Set(LOCAL_SCHEMA_V3_MANIFEST.objects.map((object) => object.name)) expect(v3Names.has("web_events")).toBe(false) expect( - LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name).filter((name) => !v3Names.has(name)), + LOCAL_SCHEMA_V4_MANIFEST.objects + .map((object) => object.name) + .filter((name) => !v3Names.has(name)), ).toEqual(["web_events", "web_events_mv"]) + + // v5 adds no objects at all — it is five trailing defaulted columns and two + // skip indexes on traces. Asserted as a column/index delta against the + // frozen v4 manifest so a stray table or a rewritten column can't ride along. + const v4Names = new Set(LOCAL_SCHEMA_V4_MANIFEST.objects.map((object) => object.name)) + expect( + LOCAL_SCHEMA_V5_MANIFEST.objects.map((object) => object.name).filter((name) => !v4Names.has(name)), + ).toEqual([]) + const v4Traces = LOCAL_SCHEMA_V4_MANIFEST.objects.find((object) => object.name === "traces") + const traces = LOCAL_SCHEMA_V5_MANIFEST.objects.find((object) => object.name === "traces") + const v4TraceColumns = new Set(v4Traces?.columns.map((column) => column.name)) + expect( + traces?.columns.map((column) => column.name).filter((name) => !v4TraceColumns.has(name)), + ).toEqual(["AiVendor", "AiSessionKeyState", "AiSessionKeyHash", "AiRulesVersion", "AiRollupHour"]) + // Every new column is default-computed: that is what keeps them out of the + // generated ingest INSERT column list until the classifier stage lands. + expect( + traces?.columns + .filter((column) => column.name.startsWith("Ai")) + .every((column) => column.defaultKind === "DEFAULT"), + ).toBe(true) + expect(traces?.columns.find((column) => column.name === "AiRollupHour")?.type).toBe("DateTime('UTC')") + expect(traces?.indexes.filter((index) => !(v4Traces?.indexes ?? []).includes(index))).toEqual([ + "idx_ai_vendor", + "idx_scope_name", + ]) }) }) @@ -110,12 +141,14 @@ describe("local migration registry", () => { "local-0001-to-0002-error-rollup", "local-0002-to-0003-service-map-ingest-bridge", "local-0003-to-0004-web-events", + "local-0004-to-0005-ai-classification-columns", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) expect(chain[1]?.to).toEqual(LOCAL_SCHEMA_V2) expect(chain[2]?.to).toEqual(LOCAL_SCHEMA_V3) expect(chain[3]?.to).toEqual(LOCAL_SCHEMA_V4) + expect(chain[4]?.to).toEqual(LOCAL_SCHEMA_V5) expect(typeof chain[0]?.apply).toBe("function") }) @@ -137,12 +170,12 @@ describe("local migration registry", () => { maple: "dev", createdAt: "2026-01-01T00:00:00.000Z", createdByMaple: "dev", - schemaVersion: 4, + schemaVersion: 5, schemaDigest: SCHEMA_DIGEST, schema: SCHEMA_FINGERPRINT, activation: "active", }), - ).toMatchObject({ version: 4, fingerprint: SCHEMA_FINGERPRINT, digest: SCHEMA_DIGEST }) + ).toMatchObject({ version: 5, fingerprint: SCHEMA_FINGERPRINT, digest: SCHEMA_DIGEST }) }) it("rejects unknown, future, downgrade, and ambiguous paths", () => { @@ -151,7 +184,7 @@ describe("local migration registry", () => { ).toThrow(/no registered/) expect(() => resolveMigrationChain( - { ...CURRENT_LOCAL_SCHEMA, version: 5, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 6, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/apps/ingest/benches/ingest_bench.rs b/apps/ingest/benches/ingest_bench.rs index fd5b99675..9faf942bd 100644 --- a/apps/ingest/benches/ingest_bench.rs +++ b/apps/ingest/benches/ingest_bench.rs @@ -15,7 +15,8 @@ use flate2::read::GzDecoder; use maple_ingest::ai_classifier::ResourceContext; use maple_ingest::ai_registry::registry; use maple_ingest::telemetry::{ - ClickHouseBreakerConfig, DatasourceNames, SamplingPolicy, TelemetryPipeline, TinybirdConfig, + AiClassificationSettings, ClickHouseBreakerConfig, DatasourceNames, SamplingPolicy, + TelemetryPipeline, TinybirdConfig, }; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}; @@ -70,6 +71,9 @@ fn bench_ingest_accept(c: &mut Criterion) { black_box(&fixture.traces), &SamplingPolicy::default(), &[], + // Classification's own cost is measured by the + // `ai_classifier` group; this one measures WAL ack. + &AiClassificationSettings::disabled(), ) .await .expect("accept traces"), diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 08ff5ec91..bb91d281c 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,12 +1,12 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a"; +pub const PROJECT_REVISION: &str = "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse // clickHouseSchemaVersion. -pub const SCHEMA_VERSION: &str = "13"; +pub const SCHEMA_VERSION: &str = "15"; pub const ORG_PLACEHOLDER: &str = "__ORG__"; #[derive(Debug)] @@ -22,9 +22,9 @@ pub const DATASOURCES: &[InsertMapping] = &[ InsertMapping { datasource: "traces", table: "traces", - columns: &["OrgId", "Timestamp", "TraceId", "SpanId", "ParentSpanId", "TraceState", "SpanName", "SpanKind", "ServiceName", "ResourceSchemaUrl", "ResourceAttributes", "ScopeSchemaUrl", "ScopeName", "ScopeVersion", "ScopeAttributes", "Duration", "StatusCode", "StatusMessage", "SpanAttributes", "EventsTimestamp", "EventsName", "EventsAttributes", "LinksTraceId", "LinksSpanId", "LinksTraceState", "LinksAttributes"], - selects: &["__ORG__", "start_time", "trace_id", "span_id", "parent_span_id", "trace_state", "span_name", "span_kind", "service_name", "resource_schema_url", "resource_attributes", "scope_schema_url", "scope_name", "scope_version", "scope_attributes", "duration", "status_code", "status_message", "span_attributes", "events_timestamp", "events_name", "events_attributes", "links_trace_id", "links_span_id", "links_trace_state", "links_attributes"], - input_schema: "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))", + columns: &["OrgId", "Timestamp", "TraceId", "SpanId", "ParentSpanId", "TraceState", "SpanName", "SpanKind", "ServiceName", "ResourceSchemaUrl", "ResourceAttributes", "ScopeSchemaUrl", "ScopeName", "ScopeVersion", "ScopeAttributes", "Duration", "StatusCode", "StatusMessage", "SpanAttributes", "EventsTimestamp", "EventsName", "EventsAttributes", "LinksTraceId", "LinksSpanId", "LinksTraceState", "LinksAttributes", "AiVendor", "AiSessionKeyState", "AiSessionKeyHash", "AiRulesVersion", "AiRollupHour"], + selects: &["__ORG__", "start_time", "trace_id", "span_id", "parent_span_id", "trace_state", "span_name", "span_kind", "service_name", "resource_schema_url", "resource_attributes", "scope_schema_url", "scope_name", "scope_version", "scope_attributes", "duration", "status_code", "status_message", "span_attributes", "events_timestamp", "events_name", "events_attributes", "links_trace_id", "links_span_id", "links_trace_state", "links_attributes", "ai_vendor", "ai_session_key_state", "ai_session_key_hash", "ai_rules_version", "ai_rollup_hour"], + input_schema: "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String)), ai_vendor LowCardinality(String), ai_session_key_state UInt8, ai_session_key_hash UInt64, ai_rules_version UInt32, ai_rollup_hour DateTime('UTC')", }, InsertMapping { datasource: "logs", diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index 8d8e36975..07857803c 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -43,9 +43,10 @@ use maple_ingest::session_analytics::{ derive_referrer_host, sanitize_session_event, sanitize_session_meta, }; use maple_ingest::telemetry::{ - AttributeMappingRule, ClickHouseBreakerConfig, ClickHouseTarget, ClickHouseTargetProvider, - DatasourceNames, ExportDestination, MappingOperation, MappingSourceContext, PipelineError, - SamplingPolicy, TelemetryPipeline, TelemetrySignal, TinybirdConfig, + AiClassificationSettings, AttributeMappingRule, ClickHouseBreakerConfig, ClickHouseTarget, + ClickHouseTargetProvider, DatasourceNames, ExportDestination, MappingOperation, + MappingSourceContext, PipelineError, SamplingPolicy, TelemetryPipeline, TelemetrySignal, + TinybirdConfig, }; use maple_ingest::usage_metrics::{billable_gb, usage_cardinality_view, UsageMetrics}; use moka::future::Cache; @@ -122,6 +123,12 @@ struct AppConfig { max_request_body_bytes: usize, org_max_in_flight: u64, require_tls: bool, + /// `INGEST_AI_CLASSIFICATION_ENABLED`. **Migration-window flag**, default + /// false: it exists only to ramp the write-side plan's §8 step 2 shadow + /// deploy and is removed once classification is unconditional in production + /// (§8 step 3). Read once per batch, never per span. `AiRollupHour` is + /// written on every row regardless of this flag. + ai_classification_enabled: bool, key_store_backend: KeyStoreBackend, clickhouse_encryption_key: Option<[u8; 32]>, lookup_hmac_key: String, @@ -349,8 +356,8 @@ impl AppConfig { let key_store_backend = resolve_key_store_backend()?; let clickhouse_encryption_key = match &key_store_backend { - // The Postgres key store decrypts BYO-ClickHouse credentials from - // org_clickhouse_settings, so it needs the encryption key. + // MAPLE_INGEST_KEY_ENCRYPTION_KEY, for BYO-ClickHouse credentials + // from org_clickhouse_settings, encrypted by apps/api with the same key. KeyStoreBackend::Postgres { .. } => { let raw = std::env::var("MAPLE_INGEST_KEY_ENCRYPTION_KEY") .map_err(|_| "MAPLE_INGEST_KEY_ENCRYPTION_KEY is required".to_string())?; @@ -460,6 +467,14 @@ impl AppConfig { } }; + // Default off: the flag ramps classification per-deployment during the + // migration window and is deleted afterwards. + let ai_classification_enabled = parse_bool( + "INGEST_AI_CLASSIFICATION_ENABLED", + std::env::var("INGEST_AI_CLASSIFICATION_ENABLED").ok(), + false, + )?; + // Default off — see the field doc. Set it on services that are only // reachable through Cloudflare. let trust_proxy_geo = parse_bool( @@ -478,6 +493,7 @@ impl AppConfig { max_request_body_bytes, org_max_in_flight, require_tls, + ai_classification_enabled, key_store_backend, clickhouse_encryption_key, lookup_hmac_key, @@ -1513,14 +1529,14 @@ async fn main() { // process exit, so a bad deploy never goes ready while a transient database // fault can no longer kill a healthy running fleet. let key_store_ready = Arc::new(AtomicBool::new(false)); - let store: Arc = match build_key_store(&config, Arc::clone(&key_store_ready)).await - { - Ok(store) => store, - Err(error) => { - eprintln!("Key store init error: {error}"); - std::process::exit(1); - } - }; + let store: Arc = + match build_key_store(&config, Arc::clone(&key_store_ready)).await { + Ok(store) => store, + Err(error) => { + eprintln!("Key store init error: {error}"); + std::process::exit(1); + } + }; // The Postgres key store resolves BYO-ClickHouse export targets from // org_clickhouse_settings (the Static backend has no DB to resolve from). @@ -2050,6 +2066,7 @@ async fn resolve_grpc_ingest_key( key_id: "sentinel".to_string(), self_managed: false, clickhouse_ready: false, + // Health-probe traffic is discarded, never classified. }); } @@ -2079,7 +2096,11 @@ async fn ready(State(state): State>) -> Response { if state.key_store_ready.load(Ordering::Relaxed) { (StatusCode::OK, "READY").into_response() } else { - (StatusCode::SERVICE_UNAVAILABLE, "DEGRADED: key store unavailable").into_response() + ( + StatusCode::SERVICE_UNAVAILABLE, + "DEGRADED: key store unavailable", + ) + .into_response() } } @@ -3414,7 +3435,9 @@ async fn handle_cloudflare_logpush_inner( key_id: resolved.secret_key_id.clone(), self_managed: resolved.self_managed, clickhouse_ready: resolved.clickhouse_ready, - }; + // Cloudflare Logpush is a logs-only connector; logs + // classification is v2 (write-side plan §3). + }; let decoded = DecodedPayload::Logs(request); let reservation = reserve_autumn_usage( state, @@ -4245,12 +4268,17 @@ async fn accept_native_decoded_payload( "maple.ingest.attribute_mapping_count", attribute_mappings.len(), ); + // The flag is already resolved from the env; this reads it once for + // the whole batch. + let ai = AiClassificationSettings::new(state.config.ai_classification_enabled); + Span::current().record("maple.ingest.ai.enabled", ai.enabled); pipeline .accept_traces_to( &resolved_key.org_id, request, &policy, attribute_mappings.as_slice(), + &ai, destination, ) .await @@ -4287,6 +4315,13 @@ async fn accept_native_decoded_payload( if stats.dropped > 0 { metrics::native_sampled_dropped(signal.path(), stats.dropped as u64); } + if stats.ai_spans_examined > 0 { + metrics::ai_spans_examined(signal.path(), stats.ai_spans_examined as u64); + Span::current().record( + "maple.ingest.ai.spans_examined", + stats.ai_spans_examined as u64, + ); + } Span::current().record("maple.ingest.native_rows", stats.rows as u64); Span::current().record("maple.ingest.sampled_dropped", stats.dropped as u64); Ok(()) @@ -4762,7 +4797,12 @@ impl PostgresKeyStore { WHERE k.private_key_hash = $1 LIMIT 1", &[&"__ingest_probe_no_match__"], ) - .instrument(postgres_client_span("probe", "SELECT", "org_ingest_keys", &self.target)) + .instrument(postgres_client_span( + "probe", + "SELECT", + "org_ingest_keys", + &self.target, + )) .await .map(|_| ()) .map_err(|error| format!("postgres probe query failed: {}", error_chain(&error))) @@ -4797,7 +4837,9 @@ impl KeyStore for PostgresKeyStore { &self.target, )) .await - .map_err(|error| format!("postgres fetch_ingest_key failed: {}", error_chain(&error)))?; + .map_err(|error| { + format!("postgres fetch_ingest_key failed: {}", error_chain(&error)) + })?; let Some(row) = rows.into_iter().next() else { return Ok(None); }; @@ -6196,6 +6238,51 @@ mod tests { } } + /// One Spring AI span, mirroring `telemetry.rs`'s AI fixture: an app-chosen + /// (insufficient) scope promoted by a `spring.ai.` attribute hit, with + /// spring_ai's session-granularity key present. + fn test_ai_trace_request() -> ExportTraceServiceRequest { + use opentelemetry_proto::tonic::trace::v1::{span, ResourceSpans, ScopeSpans, Span}; + let string_kv = |key: &str, value: &str| KeyValue { + key: key.to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(value.to_string())), + }), + }; + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![string_kv("service.name", "spring-ai-app")], + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: "org.springframework.boot".to_string(), + version: "4.1.0".to_string(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + spans: vec![Span { + trace_id: vec![0x77; 16], + span_id: vec![0x88; 8], + name: "chat_client".to_string(), + kind: span::SpanKind::Internal as i32, + start_time_unix_nano: 1_700_000_000_000_000_000, + end_time_unix_nano: 1_700_000_000_500_000_000, + attributes: vec![ + string_kv("spring.ai.kind", "chat_client"), + string_kv("spring.ai.chat.client.conversation.id", "sess-e2e"), + ], + ..Default::default() + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + } + fn test_headers(raw_key: &str) -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert( @@ -6213,6 +6300,18 @@ mod tests { queue_dir: PathBuf, forward_endpoint: String, routing_ttl: Duration, + ) -> AppState { + test_app_state_with_ai(store, queue_dir, forward_endpoint, routing_ttl, false).await + } + + /// `test_app_state` with `INGEST_AI_CLASSIFICATION_ENABLED` settable, so the + /// flag's real path through `AppConfig` can be exercised. + async fn test_app_state_with_ai( + store: Arc, + queue_dir: PathBuf, + forward_endpoint: String, + routing_ttl: Duration, + ai_classification_enabled: bool, ) -> AppState { let tinybird = test_tinybird_config(queue_dir); let key_store: Arc = store.clone(); @@ -6250,6 +6349,7 @@ mod tests { max_request_body_bytes: 1024 * 1024, org_max_in_flight: 100, require_tls: false, + ai_classification_enabled, key_store_backend: KeyStoreBackend::Static { org_id: "org_test".to_string(), }, @@ -6445,11 +6545,7 @@ mod tests { walk(dir) } - async fn replay_blob_test_state( - raw_key: &str, - org_id: &str, - queue_dir: PathBuf, - ) -> AppState { + async fn replay_blob_test_state(raw_key: &str, org_id: &str, queue_dir: PathBuf) -> AppState { let store = Arc::new(FakeKeyStore::default()); store.insert_private( raw_key, @@ -6519,9 +6615,9 @@ mod tests { assert_eq!(captured.content_type, "application/json"); assert_eq!(captured.content_encoding, "gzip"); assert!( - captured.authorization.starts_with( - "AWS4-HMAC-SHA256 Credential=test-access-key/" - ), + captured + .authorization + .starts_with("AWS4-HMAC-SHA256 Credential=test-access-key/"), "expected a SigV4 authorization header, got {:?}", captured.authorization ); @@ -7217,6 +7313,99 @@ mod tests { assert_eq!(plaintext, "ch-secret-123"); } + /// The whole write path with the flag on: HTTP request → key resolve → + /// `AppConfig.ai_classification_enabled` → classification → the NDJSON body + /// ClickHouse actually receives. + #[tokio::test] + async fn ai_classification_flag_reaches_the_clickhouse_row() { + let (ch_tx, mut ch_rx) = tokio::sync::mpsc::unbounded_channel(); + let ch_listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let ch_addr = ch_listener.local_addr().unwrap(); + let ch_app = Router::new() + .route("/", post(fake_clickhouse_import)) + .with_state(ch_tx); + tokio::spawn(async move { + axum::serve(ch_listener, ch_app).await.unwrap(); + }); + + let queue_dir = unique_main_test_dir("ai-classification"); + let store = Arc::new(FakeKeyStore::default()); + let raw_key = "maple_sk_test_ai_flag"; + store.insert_private( + raw_key, + KeyRow { + org_id: "org_ai_flag".to_string(), + self_managed: true, + clickhouse_ready: true, + }, + ); + store.insert_clickhouse_target( + "org_ai_flag", + ClickHouseTargetRow { + ch_url: format!("http://{ch_addr}"), + ch_user: "ingest".to_string(), + ch_password_ciphertext: None, + ch_password_iv: None, + ch_password_tag: None, + ch_database: "maple".to_string(), + schema_version: CLICKHOUSE_SCHEMA_VERSION.to_string(), + }, + ); + let state = test_app_state_with_ai( + Arc::clone(&store), + queue_dir.clone(), + "http://127.0.0.1:1".to_string(), + Duration::from_millis(5), + true, + ) + .await; + + let (response, item_count, _, _, _) = handle_signal_inner( + &state, + &test_headers(raw_key), + Bytes::from(test_ai_trace_request().encode_to_vec()), + Signal::Traces, + ) + .await + .expect("request should be accepted through the native ClickHouse path"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(item_count, 1); + + let import = tokio::time::timeout(Duration::from_secs(2), ch_rx.recv()) + .await + .expect("ready org should write to ClickHouse") + .expect("ClickHouse channel should stay open"); + + // The INSERT must name the five columns, or the values below would land + // in the wrong ones. + assert!(import.query.contains("AiVendor"), "{}", import.query); + assert!(import.query.contains("AiRollupHour"), "{}", import.query); + + let row: serde_json::Value = serde_json::from_str(import.body.trim()).expect("NDJSON row"); + assert_eq!(row["ai_vendor"], "spring_ai"); + assert_eq!(row["ai_session_key_state"], serde_json::json!(6)); + assert_eq!( + row["ai_session_key_hash"], + serde_json::json!(maple_ingest::cityhash102::city_hash64(b"sess-e2e")) + ); + assert_ne!(row["ai_rules_version"], serde_json::json!(0)); + // The fixture's span start is a fixed 2023 timestamp, i.e. far outside + // the [receive − 7d, receive + 1d] window, so the clamp must relocate it + // to the hour this batch was received rather than minting a 2023 + // partition. (The clamp's window arithmetic is covered exhaustively in + // `telemetry.rs` against an injected receive time.) + let now_secs = (current_time_millis() / 1000) as i64; + let this_hour = chrono::DateTime::from_timestamp(now_secs - now_secs.rem_euclid(3600), 0) + .expect("valid receive hour") + .format("%Y-%m-%d %H:%M:%S") + .to_string(); + assert_eq!(row["ai_rollup_hour"], serde_json::json!(this_hour)); + + let _ = std::fs::remove_dir_all(queue_dir); + } + #[test] fn schema_revision_accepts_newer_schemas_and_rejects_older_ones() { let needed = required_schema_revision(); diff --git a/apps/ingest/src/metrics.rs b/apps/ingest/src/metrics.rs index 602c6a634..3187dc54c 100644 --- a/apps/ingest/src/metrics.rs +++ b/apps/ingest/src/metrics.rs @@ -141,6 +141,13 @@ static NATIVE_ROWS_TOTAL: LazyLock> = LazyLock::new(|| { .build() }); +static AI_SPANS_EXAMINED_TOTAL: LazyLock> = LazyLock::new(|| { + METER + .u64_counter("ingest_ai_spans_examined_total") + .with_description("Spans the AI classifier examined, by signal") + .build() +}); + static NATIVE_SAMPLED_DROPPED_TOTAL: LazyLock> = LazyLock::new(|| { METER .u64_counter("ingest_native_sampled_dropped_total") @@ -588,6 +595,18 @@ pub fn native_rows(signal: &str, count: u64) { NATIVE_ROWS_TOTAL.add(count, &[KeyValue::new("signal", signal.to_string())]); } +/// Spans the AI classifier examined — write-side plan §4's completeness signal. +/// +/// Labeled by `signal` only, exactly like `native_rows`, so the two are directly +/// comparable: with classification enabled every accepted trace row is an +/// examined span, and any divergence between these series is a bug (a code path +/// building rows without classifying, or a partially-flagged fleet). It is +/// deliberately *not* org-labeled — this counter moves constantly on a healthy +/// fleet, which is the opposite of `org_data_loss`'s cardinality budget. +pub fn ai_spans_examined(signal: &str, count: u64) { + AI_SPANS_EXAMINED_TOTAL.add(count, &[KeyValue::new("signal", signal.to_string())]); +} + /// Rows dropped by sampling in the native pipeline. pub fn native_sampled_dropped(signal: &str, count: u64) { NATIVE_SAMPLED_DROPPED_TOTAL.add(count, &[KeyValue::new("signal", signal.to_string())]); diff --git a/apps/ingest/src/otel.rs b/apps/ingest/src/otel.rs index de50e880c..7902bd52d 100644 --- a/apps/ingest/src/otel.rs +++ b/apps/ingest/src/otel.rs @@ -382,6 +382,12 @@ pub fn accept_internal_span(signal_path: &'static str, destination: &'static str "maple.ingest.destination" = destination, "maple.ingest.native_rows" = Empty, "maple.ingest.sampled_dropped" = Empty, + // Whether the migration-window classification flag was on for this + // batch, and how many spans it examined. Batch-level, not per span: + // per-span classification spans on this hot path are exactly what + // CLAUDE.md's self-observability rule forbids. + "maple.ingest.ai.enabled" = Empty, + "maple.ingest.ai.spans_examined" = Empty, ) } diff --git a/apps/ingest/src/telemetry.rs b/apps/ingest/src/telemetry.rs index 8c447b1da..a3330e1c1 100644 --- a/apps/ingest/src/telemetry.rs +++ b/apps/ingest/src/telemetry.rs @@ -7,6 +7,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; +use crate::ai_classifier::{self, ResourceContext, SpanClassification}; +use crate::ai_registry::registry; use crate::clickhouse_insert_mappings::{self, InsertMapping}; use crate::metrics; use crate::otel::{ @@ -602,6 +604,129 @@ impl PipelineError { pub struct AcceptStats { pub rows: usize, pub dropped: usize, + /// Spans the AI classifier examined in this batch. Zero when the migration + /// flag is off; otherwise equal to `rows` for traces (plan §4's + /// `spans_ingested` vs `spans_examined` completeness pair). + pub ai_spans_examined: usize, +} + +/// Per-batch inputs for AI classification and rollup-hour clamping. +/// +/// Built once per accepted payload, never per span: the flag is read once and +/// `receive_time_secs` is captured a single time so every span in one payload +/// clamps against the same instant. +#[derive(Clone, Debug)] +pub struct AiClassificationSettings { + /// `INGEST_AI_CLASSIFICATION_ENABLED`. **Migration-window flag only** — the + /// write-side plan (§8 step 3) removes it once classification is at 100% for + /// a full clock hour and the rollup MV exists; production then classifies + /// unconditionally. `AiRollupHour` is written whether it is set or not. + pub enabled: bool, + /// Batch receive time, epoch seconds. + pub receive_time_secs: i64, +} + +impl AiClassificationSettings { + pub fn new(enabled: bool) -> Self { + Self { + enabled, + receive_time_secs: unix_now_secs(), + } + } + + /// Flag off. `AiRollupHour` is still computed and written. + pub fn disabled() -> Self { + Self::new(false) + } + + /// Explicit receive time, for tests that assert the clamp windows. + #[cfg(test)] + pub fn at(enabled: bool, receive_time_secs: i64) -> Self { + Self { + enabled, + receive_time_secs, + } + } +} + +fn unix_now_secs() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs() as i64) + .unwrap_or(0) +} + +/// The four classification columns as the row writes them. Extracted from a +/// [`SpanClassification`] immediately so the row builder never has to hold a +/// borrow of the span's attribute list. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct AiRowFields { + vendor: &'static str, + session_state: u8, + session_key_hash: u64, + rules_version: u32, +} + +impl AiRowFields { + /// Flag-off values. `rules_version = 0` is the plan's pre-rollout marker: + /// `AiRulesVersion = 0` means "never examined", which is what a flag-off row + /// is, and is distinguishable from an examined-and-non-AI row (non-zero + /// version, empty vendor). + const UNEXAMINED: Self = Self { + vendor: "", + session_state: ai_classifier::session_state::NOT_EXAMINED, + session_key_hash: 0, + rules_version: 0, + }; + + fn from_classification(classification: &SpanClassification<'_>) -> Self { + Self { + vendor: classification.vendor_slug(), + session_state: classification.session_state, + session_key_hash: classification.session_key_hash(), + rules_version: classification.rules_version, + } + } +} + +/// Seconds in the clamp window's past and future halves (write-side plan §3). +const ROLLUP_CLAMP_PAST_SECS: i64 = 7 * 24 * 60 * 60; +const ROLLUP_CLAMP_FUTURE_SECS: i64 = 24 * 60 * 60; + +/// `AiRollupHour`: `toStartOfHour(Timestamp)` when the span's start time is +/// within `[receive − 7d, receive + 1d]`, else `toStartOfHour(receive_time)`. +/// +/// Client timestamps are attacker- and replay-controlled, and this column is the +/// rollup's partition key, so an unclamped value means unbounded partition +/// creation and rows whose TTL never fires. Clamping at write time rather than +/// in the MV is the only deterministic option: an MV-side clamp would need +/// `now()`, which a later partition rebuild re-evaluates at rebuild time and +/// would silently relocate rows across hours. +fn rollup_hour_secs(span_start_unix_nano: u64, receive_time_secs: i64) -> i64 { + let span_secs = (span_start_unix_nano / 1_000_000_000) as i64; + let in_window = span_secs >= receive_time_secs - ROLLUP_CLAMP_PAST_SECS + && span_secs <= receive_time_secs + ROLLUP_CLAMP_FUTURE_SECS; + let chosen = if in_window { + span_secs + } else { + receive_time_secs + }; + chosen - chosen.rem_euclid(3600) +} + +/// `DateTime('UTC')` wire format: `YYYY-MM-DD HH:MM:SS`, the second-precision +/// sibling of `format_timestamp_nano`'s `DateTime64(9)` rendering. +/// +/// A string, not epoch seconds. Both destinations accept either — Tinybird's +/// JSONPath ingestion and ClickHouse's `input('… AiRollupHour DateTime(\'UTC\')')` +/// both parse a quoted datetime — but every other timestamp this row writer +/// emits is a string in this shape, and a lone numeric column would make the +/// JSON row's timestamps inconsistent to read and to diff. +fn format_datetime_secs(unix_secs: i64) -> String { + match chrono::DateTime::from_timestamp(unix_secs, 0) { + Some(dt) => dt.format("%Y-%m-%d %H:%M:%S").to_string(), + None => "1970-01-01 00:00:00".to_string(), + } } #[derive(Clone)] @@ -721,12 +846,14 @@ impl TelemetryPipeline { request: &ExportTraceServiceRequest, sampling_policy: &SamplingPolicy, attribute_mappings: &[AttributeMappingRule], + ai: &AiClassificationSettings, ) -> Result { self.accept_traces_to( org_id, request, sampling_policy, attribute_mappings, + ai, ExportDestination::Tinybird, ) .await @@ -738,6 +865,7 @@ impl TelemetryPipeline { request: &ExportTraceServiceRequest, sampling_policy: &SamplingPolicy, attribute_mappings: &[AttributeMappingRule], + ai: &AiClassificationSettings, destination: ExportDestination, ) -> Result { let (frames, stats) = { @@ -749,6 +877,7 @@ impl TelemetryPipeline { request, sampling_policy, attribute_mappings, + ai, )?; record_encode_stats(&span, &frames, &stats); (frames, stats) @@ -841,6 +970,8 @@ impl TelemetryPipeline { let stats = AcceptStats { rows: rows.len(), dropped: 0, + // Session-replay rows, not spans — nothing to classify. + ai_spans_examined: 0, }; let frames = rows_to_frames(org_id, hash64(org_id), signal, datasource, rows); self.commit_frames(frames, destination).await?; @@ -2183,23 +2314,38 @@ fn encode_traces( request: &ExportTraceServiceRequest, policy: &SamplingPolicy, attribute_mappings: &[AttributeMappingRule], + ai: &AiClassificationSettings, ) -> Result<(Vec, AcceptStats), PipelineError> { let mut rows = Vec::with_capacity(count_trace_rows(request)); let mut dropped = 0usize; + let mut ai_spans_examined = 0usize; let mut routing_key = hash64(org_id); let sample_ratio = policy.clamped_ratio(); let sample_rate = 1.0 / sample_ratio; + // Rules can add, rename or delete registry-referenced span attributes, so a + // span classified from the wire list would not be the span the row stores. + // With no rules configured — the overwhelmingly common case — the Map is a + // faithful canonicalization of the wire list, so the wire list *is* the row. + let remapped = !attribute_mappings.is_empty(); for resource_spans in &request.resource_spans { let resource = resource_spans.resource.as_ref(); - let resource_attrs = resource - .map(|resource| attr_map(&resource.attributes)) - .unwrap_or_default(); + let resource_attributes = resource + .map(|resource| resource.attributes.as_slice()) + .unwrap_or(&[]); + let resource_attrs = attr_map(resource_attributes); let service_name = resource_attrs .get("service.name") .and_then(Value::as_str) .unwrap_or("") .to_string(); + // Hoisted once per ResourceSpans, per the plan's §2 batch algorithm — + // but only when a span will actually use it. With the flag off there is + // nothing to classify, and with mapping rules configured each span needs + // its own context over the rewritten list, so hoisting either way would + // be pure waste on the default (flag-off) path. + let ai_resource = (ai.enabled && !remapped) + .then(|| ResourceContext::new(registry(), resource_attributes)); for scope_spans in &resource_spans.scope_spans { let scope = scope_spans.scope.as_ref(); @@ -2208,6 +2354,10 @@ fn encode_traces( .unwrap_or_default(); let scope_name = scope.map(|scope| scope.name.as_str()).unwrap_or(""); let scope_version = scope.map(|scope| scope.version.as_str()).unwrap_or(""); + // Hoisted once per ScopeSpans; every span below reuses it. + let ai_scope = ai_resource + .as_ref() + .map(|resource| resource.scope(scope, &scope_spans.schema_url)); for span in &scope_spans.spans { let trace_id = bytes_hex(&span.trace_id); @@ -2233,6 +2383,39 @@ fn encode_traces( } apply_attribute_mappings(attribute_mappings, &resource_attrs, &mut span_attrs); + // Classification runs here, after remapping, so it sees exactly + // the attribute map the row persists (plan §6). Flag-off skips + // the work entirely and writes the pre-rollout marker values; + // `AiRollupHour` below is written either way. + let ai_fields = if !ai.enabled { + AiRowFields::UNEXAMINED + } else { + ai_spans_examined += 1; + if remapped { + // The hoisted context borrows the wire attribute list; + // the rewritten list is per-span, so the (cheap) context + // is rebuilt over it. Only orgs with mapping rules pay. + let rewritten = key_values_from_map(&span_attrs); + let resource_context = + ResourceContext::new(registry(), resource_attributes); + let scope_context = resource_context.scope(scope, &scope_spans.schema_url); + AiRowFields::from_classification( + &scope_context.classify_span(&span.name, &rewritten), + ) + } else { + let scope_context = ai_scope + .as_ref() + .expect("hoisted whenever classification reads the wire attributes"); + AiRowFields::from_classification( + &scope_context.classify_span(&span.name, &span.attributes), + ) + } + }; + let ai_rollup_hour = format_datetime_secs(rollup_hour_secs( + span.start_time_unix_nano, + ai.receive_time_secs, + )); + let events_timestamp: Vec = span .events .iter() @@ -2291,7 +2474,12 @@ fn encode_traces( "links_trace_id": links_trace_id, "links_span_id": links_span_id, "links_trace_state": links_trace_state, - "links_attributes": links_attributes + "links_attributes": links_attributes, + "ai_vendor": ai_fields.vendor, + "ai_session_key_state": ai_fields.session_state, + "ai_session_key_hash": ai_fields.session_key_hash, + "ai_rules_version": ai_fields.rules_version, + "ai_rollup_hour": ai_rollup_hour }))?); } } @@ -2300,6 +2488,7 @@ fn encode_traces( let stats = AcceptStats { rows: rows.len(), dropped, + ai_spans_examined, }; let frames = rows_to_frames( org_id, @@ -2360,6 +2549,8 @@ fn encode_logs( let stats = AcceptStats { rows: rows.len(), dropped: 0, + // Logs classification is v2 (write-side plan §3). + ai_spans_examined: 0, }; let frames = rows_to_frames( org_id, @@ -2573,6 +2764,8 @@ fn encode_metrics( AcceptStats { rows: row_count, dropped: 0, + // Metrics classification is v2 (write-side plan §3). + ai_spans_examined: 0, }, )) } @@ -2783,17 +2976,56 @@ fn format_sample_rate(sample_rate: f64) -> String { } } +/// The written span-attribute Map back as an attribute list, so the classifier +/// can be pointed at the post-remapping row. +/// +/// Only built for orgs that actually configured attribute-mapping rules. Values +/// are already canonical strings (they came out of [`any_value_string`]), so +/// re-wrapping them as `StringValue` round-trips exactly. +fn key_values_from_map(attributes: &Map) -> Vec { + attributes + .iter() + .map(|(key, value)| KeyValue { + key: key.clone(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + value.as_str().unwrap_or_default().to_string(), + )), + }), + }) + .collect() +} + +/// Canonicalizes an attribute list into the row's Map column. +/// +/// **Duplicate keys.** A plain `insert` loop keeps the *last* occurrence, and +/// that is still what happens for keys no classification rule consults. For +/// **registry-referenced** keys the first occurrence wins instead, because the +/// classifier dedupes those first-occurrence-wins (write-side plan §2) and the +/// §6 alignment contract requires SQL over the written row to reproduce the Rust +/// verdict — if the Map kept the last `gen_ai.system` while the matcher read the +/// first, a rollup rebuild would disagree with the stored `AiVendor`. +/// +/// The registry probe only runs on an actual collision, so the ordinary path +/// (no duplicate keys) costs exactly what it did before. fn attr_map(attributes: &[KeyValue]) -> Map { let mut out = Map::with_capacity(attributes.len()); for attribute in attributes { - out.insert( - attribute.key.clone(), - json!(attribute - .value - .as_ref() - .map(any_value_string) - .unwrap_or_default()), - ); + let value = json!(attribute + .value + .as_ref() + .map(any_value_string) + .unwrap_or_default()); + match out.entry(attribute.key.clone()) { + serde_json::map::Entry::Vacant(slot) => { + slot.insert(value); + } + serde_json::map::Entry::Occupied(mut slot) => { + if !registry().references_key(&attribute.key) { + slot.insert(value); + } + } + } } out } @@ -3310,6 +3542,7 @@ mod tests { &request, &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); assert_eq!(stats.rows, 1); @@ -3626,6 +3859,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ExportDestination::ClickHouse, ) .await @@ -4345,6 +4579,11 @@ mod tests { "links_span_id", "links_trace_state", "links_attributes", + "ai_vendor", + "ai_session_key_state", + "ai_session_key_hash", + "ai_rules_version", + "ai_rollup_hour", ]; const METRIC_COMMON: &[&str] = &[ @@ -4508,6 +4747,330 @@ mod tests { } } + // ----------------------------------------------------------------------- + // AI classification (write-side plan §2/§3) + // ----------------------------------------------------------------------- + + /// Wall-clock instant every AI test clamps against: 2023-11-14 22:13:20 UTC, + /// the same second `populated_trace_request`'s span starts at. + const AI_RECEIVE_SECS: i64 = 1_700_000_000; + + /// spring_ai's only session-key candidate, authoritative on spans with + /// `spring.ai.kind = chat_client`. + const SPRING_AI_SESSION_KEY: &str = "spring.ai.chat.client.conversation.id"; + + /// A Spring AI span: a scope that is only a *conditional* candidate + /// (`org.springframework.boot` is app-chosen, so insufficient on its own), + /// promoted by the `spring.ai.` attribute hit, carrying spring_ai's + /// session-granularity key on an authoritative `chat_client` span. + fn ai_trace_request( + span_attributes: Vec, + start_unix_nano: u64, + ) -> ExportTraceServiceRequest { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![ + string_kv("service.name", "spring-ai-app"), + string_kv("maple_org_id", "org_ai"), + ], + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: "org.springframework.boot".to_string(), + version: "4.1.0".to_string(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + spans: vec![Span { + trace_id: vec![0x44; 16], + span_id: vec![0x55; 8], + name: "chat_client".to_string(), + kind: span::SpanKind::Internal as i32, + start_time_unix_nano: start_unix_nano, + end_time_unix_nano: start_unix_nano + 1_000_000, + attributes: span_attributes, + ..Default::default() + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + } + + fn spring_ai_attributes() -> Vec { + vec![ + string_kv("spring.ai.kind", "chat_client"), + string_kv("gen_ai.operation.name", "chat"), + string_kv(SPRING_AI_SESSION_KEY, "sess-abc"), + ] + } + + fn encode_ai_row(request: &ExportTraceServiceRequest, ai: &AiClassificationSettings) -> Value { + let (frames, stats) = encode_traces( + &test_cfg().datasources, + "org_ai", + request, + &SamplingPolicy::default(), + &[], + ai, + ) + .unwrap(); + assert_eq!(stats.rows, 1); + frame_row(&frames[0]) + } + + #[test] + fn classification_on_writes_vendor_state_hash_and_version() { + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let row = encode_ai_row(&request, &ai); + + assert_eq!(row["ai_vendor"], "spring_ai"); + assert_eq!( + row["ai_session_key_state"], + json!(ai_classifier::session_state::SESSION) + ); + assert_eq!(row["ai_rules_version"], json!(registry().version())); + // The construction the SQL leg reproduces: cityHash64 of the bare value. + let expected = crate::cityhash102::city_hash64(b"sess-abc"); + assert_ne!(expected, 0); + assert_eq!(row["ai_session_key_hash"], json!(expected)); + assert_eq!(row["ai_rollup_hour"], "2023-11-14 22:00:00"); + } + + #[test] + fn classification_examined_count_matches_row_count() { + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let (_, on) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + &[], + &AiClassificationSettings::at(true, AI_RECEIVE_SECS), + ) + .unwrap(); + assert_eq!(on.ai_spans_examined, on.rows); + + let (_, off) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + &[], + &AiClassificationSettings::at(false, AI_RECEIVE_SECS), + ) + .unwrap(); + assert_eq!(off.ai_spans_examined, 0); + } + + #[test] + fn classification_off_writes_zeros_but_still_writes_the_rollup_hour() { + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let ai = AiClassificationSettings::at(false, AI_RECEIVE_SECS); + let row = encode_ai_row(&request, &ai); + + assert_eq!(row["ai_vendor"], ""); + assert_eq!(row["ai_session_key_state"], json!(0)); + assert_eq!(row["ai_session_key_hash"], json!(0)); + // 0, not the registry version: a flag-off row is "never examined". + assert_eq!(row["ai_rules_version"], json!(0)); + assert_eq!(row["ai_rollup_hour"], "2023-11-14 22:00:00"); + } + + #[test] + fn a_non_ai_span_is_examined_and_stamped_but_carries_no_vendor() { + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let (frames, stats) = encode_traces( + &test_cfg().datasources, + "org_contract", + &populated_trace_request(), + &SamplingPolicy::default(), + &[], + &ai, + ) + .unwrap(); + let row = frame_row(&frames[0]); + assert_eq!(stats.ai_spans_examined, 1); + assert_eq!(row["ai_vendor"], ""); + assert_eq!(row["ai_session_key_state"], json!(0)); + // Non-zero version with an empty vendor is the plan's "definitively + // classified non-AI" marker — the whole point of stamping non-AI spans. + assert_eq!(row["ai_rules_version"], json!(registry().version())); + } + + #[test] + fn rollup_hour_clamps_stale_and_future_timestamps_to_receive_time() { + let receive_hour = "2023-11-14 22:00:00"; + let ai = || AiClassificationSettings::at(true, AI_RECEIVE_SECS); + + // In window, one hour before receive: the span's own hour is kept. + let in_window = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 - 3_600) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&in_window, &ai())["ai_rollup_hour"], + "2023-11-14 21:00:00" + ); + + // In window, at the far edge of the past half (exactly 7 days back). + let edge_past = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 - 7 * 86_400) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&edge_past, &ai())["ai_rollup_hour"], + "2023-11-07 22:00:00" + ); + + // Older than 7 days: clamped to the receive hour. + let too_old = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 - 8 * 86_400) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&too_old, &ai())["ai_rollup_hour"], + receive_hour + ); + + // Further ahead than 1 day: clamped too. This is the replay/attacker case. + let too_new = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 + 86_400 + 3_600) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&too_new, &ai())["ai_rollup_hour"], + receive_hour + ); + + // A zero timestamp is 1970 — far outside the window, so it clamps too + // rather than creating a 1970 partition. + let epoch_zero = ai_trace_request(spring_ai_attributes(), 0); + assert_eq!( + encode_ai_row(&epoch_zero, &ai())["ai_rollup_hour"], + receive_hour + ); + } + + #[test] + fn rollup_hour_matches_the_clickhouse_datetime_format() { + // `DateTime('UTC')` parses `YYYY-MM-DD HH:MM:SS` — 19 chars, no + // fractional part (that is `format_timestamp_nano`'s DateTime64(9)). + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let row = encode_ai_row( + &request, + &AiClassificationSettings::at(false, AI_RECEIVE_SECS), + ); + let hour = row["ai_rollup_hour"].as_str().unwrap(); + assert_eq!(hour.len(), 19, "not DateTime('UTC'): {hour:?}"); + assert!(hour.ends_with(":00:00"), "not an hour boundary: {hour:?}"); + let parsed = chrono::NaiveDateTime::parse_from_str(hour, "%Y-%m-%d %H:%M:%S") + .expect("ClickHouse DateTime literal must round-trip"); + assert_eq!(parsed.and_utc().timestamp() % 3600, 0); + // The generated insert mapping declares the leaf, so ClickHouse's + // `input()` schema is what parses this string. + let traces = clickhouse_insert_mappings::DATASOURCES + .iter() + .find(|mapping| mapping.datasource == "traces") + .unwrap(); + assert!(traces.columns.contains(&"AiRollupHour")); + assert!(traces + .input_schema + .contains("ai_rollup_hour DateTime('UTC')")); + assert!(traces.columns.contains(&"AiVendor")); + assert!(traces.columns.contains(&"AiSessionKeyState")); + assert!(traces.columns.contains(&"AiSessionKeyHash")); + assert!(traces.columns.contains(&"AiRulesVersion")); + } + + #[test] + fn duplicate_registry_keys_resolve_the_same_way_in_the_row_and_the_classifier() { + // Two session-key attributes on one span. The classifier takes the + // first; the written Map must agree, or a SQL rebuild over the row would + // hash a different session than the stored `AiSessionKeyHash`. + let mut attributes = spring_ai_attributes(); + attributes.push(string_kv(SPRING_AI_SESSION_KEY, "sess-SECOND")); + // A non-registry duplicate keeps the historical last-wins behaviour. + attributes.push(string_kv("http.route", "/first")); + attributes.push(string_kv("http.route", "/second")); + + let request = ai_trace_request(attributes, AI_RECEIVE_SECS as u64 * 1_000_000_000); + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let row = encode_ai_row(&request, &ai); + + assert_eq!(row["span_attributes"][SPRING_AI_SESSION_KEY], "sess-abc"); + assert_eq!(row["span_attributes"]["http.route"], "/second"); + // The hash is over the value the row stores, not the last one on the wire. + assert_eq!( + row["ai_session_key_hash"], + json!(crate::cityhash102::city_hash64(row["span_attributes"][SPRING_AI_SESSION_KEY] + .as_str() + .unwrap() + .as_bytes() + )) + ); + } + + #[test] + fn classification_reads_the_row_after_attribute_remapping() { + // The org moves a custom key onto spring_ai's session-key attribute. + // Classifying the wire attributes would miss it and write state 3; + // classifying the written row resolves it (plan §6). + let attributes = vec![ + string_kv("spring.ai.kind", "chat_client"), + string_kv("app.conversation", "sess-remapped"), + ]; + let request = ai_trace_request(attributes, AI_RECEIVE_SECS as u64 * 1_000_000_000); + let rules = [AttributeMappingRule { + source_context: MappingSourceContext::Span, + source_key: "app.conversation".to_string(), + target_key: SPRING_AI_SESSION_KEY.to_string(), + operation: MappingOperation::Move, + }]; + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let (frames, _) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + &rules, + &ai, + ) + .unwrap(); + let row = frame_row(&frames[0]); + + assert_eq!( + row["span_attributes"][SPRING_AI_SESSION_KEY], + "sess-remapped" + ); + assert_eq!( + row["ai_session_key_state"], + json!(ai_classifier::session_state::SESSION) + ); + assert_eq!( + row["ai_session_key_hash"], + json!(crate::cityhash102::city_hash64(b"sess-remapped" + )) + ); + } + fn one_of_each_metric_request() -> ExportMetricsServiceRequest { let base = NumberDataPoint { attributes: vec![string_kv("route", "/checkout")], @@ -4639,6 +5202,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); let row = frame_row(&frames[0]); @@ -4714,6 +5278,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); let trace_row = frame_row(&trace_frames[0]); @@ -4768,6 +5333,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); assert_eq!(trace_frames[0].datasource, "tenant_traces_v2"); @@ -4946,7 +5512,13 @@ mod tests { let request = populated_trace_request(); let stats = pipeline - .accept_traces("org_contract", &request, &SamplingPolicy::default(), &[]) + .accept_traces( + "org_contract", + &request, + &SamplingPolicy::default(), + &[], + &AiClassificationSettings::disabled(), + ) .await .unwrap(); assert_eq!(stats.rows, 1); diff --git a/packages/domain/src/clickhouse/migrations/0015_ai_classification_columns.ts b/packages/domain/src/clickhouse/migrations/0015_ai_classification_columns.ts new file mode 100644 index 000000000..92e781890 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0015_ai_classification_columns.ts @@ -0,0 +1,60 @@ +/** + * Migration 0015 — AI classification columns on `traces`. + * + * Each column is appended at the end of the table and carries a constant + * DEFAULT, so the `ALTER` is metadata-only on ClickHouse and Tinybird alike and + * existing parts read the default for free. No `MATERIALIZE COLUMN` is needed + * for the same reason, and rows written before the classifier shipped stay + * readable (`AiRulesVersion = 0`). + * + * Column contracts (mirrored in `datasources.ts`, keep them in sync): + * - `AiVendor` '' means "not classified as AI", not "unknown vendor". + * - `AiRulesVersion` 0 means the row predates classification; any non-zero + * value means the span was examined, including a non-AI verdict. This is the + * only way to distinguish "not AI" from "never looked at". + * - `AiSessionKeyState` is a frozen quality enum 0-6. + * - `AiSessionKeyHash` is `cityHash64(value)`, 0 unless state >= 5. + * - `AiRollupHour` is a receive-time-clamped rollup hour written by ingest, + * epoch 0 until the ingest stage lands. + * + * `requiredForIngest: true` (it was `false` while the columns had no producer). + * The gateway now names all five in its INSERT column list, so a BYO cluster + * missing them would reject every direct insert — `clickHouseSchemaVersion` + * therefore has to become "15" so the routing gate catches that org first. + * + * What an unmigrated BYO org experiences: `fetch_ingest_key`'s + * `SCHEMA_REVISION_COMPATIBLE_SQL` compares the org's stamped + * `org_clickhouse_settings.schema_version` against this value numerically, so a + * cluster still at 14 resolves `clickhouse_ready = false` and its traffic routes + * to the managed pipeline instead of its own cluster. No data is lost and no + * insert fails; the org silently falls back until `applySchema` stamps 15, at + * which point routing returns on the next 30s cache TTL. This is the designed + * mechanism, not a side effect — contrast migrations 0010/0014, which changed + * nothing the gateway writes and so stayed `requiredForIngest: false`. + * + * The two skip indexes are declared on the datasource as well — that is what + * puts them on managed (Tinybird) orgs and freshly bootstrapped clusters; these + * statements backfill clusters already at version 14. Deliberately no + * `MATERIALIZE INDEX`: that is a mutation over the whole table, and the 30-day + * TTL rolls every unindexed part out on its own. + * + * Both index types are ClickHouse 24.12-compatible. `set(0)` is unbounded on + * purpose (the vendor allowlist is closed at ~30 values, and a capped set that + * overflows silently degrades to always-match); `ScopeName` gets tokenbf_v1 + * rather than bloom_filter because the registry's scope matchers include prefix + * rules. + */ +export const migration_0015_ai_classification_columns = { + version: 15, + description: "Add AI classification columns and vendor/scope skip indexes to traces", + requiredForIngest: true, + statements: [ + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiVendor LowCardinality(String) DEFAULT ''", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiSessionKeyState UInt8 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiSessionKeyHash UInt64 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiRulesVersion UInt32 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiRollupHour DateTime('UTC') DEFAULT toDateTime(0)", + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4", + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 5bfbe9183..7444d93b7 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -17,6 +17,7 @@ import { migration_0011_session_analytics_columns } from "./0011_session_analyti import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events, webEventsBackfill } from "./0014_web_events" +import { migration_0015_ai_classification_columns } from "./0015_ai_classification_columns" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -31,15 +32,49 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) - expect(migrations.at(-1)).toBe(migration_0014_web_events) - expect(latestMigrationVersion).toBe(14) - // 0010 and 0014 are performance-only, so the ingest-gating version skips - // both and stays at 13 — nothing writes `web_events` directly, and bumping - // it would un-ready every BYO-CH org's ingest routing for a read-path change. - expect(clickHouseSchemaVersion).toBe("13") + expect(migrations.map((m) => m.version)).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + ]) + expect(migrations.at(-1)).toBe(migration_0015_ai_classification_columns) + expect(latestMigrationVersion).toBe(15) + // 0010 and 0014 are performance/storage-only, so the ingest-gating version + // skips both — nothing writes `web_events` directly and search indexes + // change nothing the gateway sends, and bumping for either would un-ready + // every BYO-CH org's routing for a change their ingest path never needs. expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) + + // 0015 is different: the gateway's INSERT now names all five AI columns, + // so a BYO cluster without them would reject every direct insert. Gating + // on it is the designed fallback — an org stamped below 15 resolves + // `clickhouse_ready = false` and routes to the managed pipeline until its + // schema syncs. + expect(migration_0015_ai_classification_columns.requiredForIngest).toBe(true) + expect(clickHouseSchemaVersion).toBe("15") + }) + + it("adds the AI classification columns as defaulted trailing columns with no mutation", () => { + const sql = migration_0015_ai_classification_columns.statements.join("\n") + + // Every column defaulted: that is what makes the ALTER metadata-only and + // keeps rows written before the classifier shipped readable + // (`AiRulesVersion = 0` = never examined). + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiVendor LowCardinality(String) DEFAULT ''") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiSessionKeyState UInt8 DEFAULT 0") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiSessionKeyHash UInt64 DEFAULT 0") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiRulesVersion UInt32 DEFAULT 0") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiRollupHour DateTime('UTC') DEFAULT toDateTime(0)") + expect(sql).toContain("ADD INDEX IF NOT EXISTS idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4") + expect(sql).toContain( + "ADD INDEX IF NOT EXISTS idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4", + ) + + // Nothing here may rewrite parts: the 30-day TTL retires unindexed parts on + // its own, and a whole-table mutation on `traces` is the expensive mistake. + expect(sql).not.toContain("MATERIALIZE INDEX") + expect(sql).not.toContain("MATERIALIZE COLUMN") + expect(sql).not.toContain("OPTIMIZE TABLE") + expect(migration_0015_ai_classification_columns.statements.filter(isBackfill)).toHaveLength(0) }) it("installs web_events with a live-write MV and no POPULATE", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index b719e9d98..d6742a6f3 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -13,6 +13,7 @@ import { migration_0011_session_analytics_columns } from "./0011_session_analyti import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events } from "./0014_web_events" +import { migration_0015_ai_classification_columns } from "./0015_ai_classification_columns" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -58,6 +59,7 @@ export const migrations: ReadonlyArray = [ migration_0012_session_event_attribute_keys, migration_0013_service_map_ingest_bridge, migration_0014_web_events, + migration_0015_ai_classification_columns, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 8db9cd4ff..92a9bac8b 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" as const +export const projectRevision = "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -38,7 +38,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS trace_detail_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, SpanId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS trace_list_mv (\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL Timestamp + INTERVAL 30 DAY", - "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n AiVendor LowCardinality(String) DEFAULT '',\n AiSessionKeyState UInt8 DEFAULT 0,\n AiSessionKeyHash UInt64 DEFAULT 0,\n AiRulesVersion UInt32 DEFAULT 0,\n AiRollupHour DateTime('UTC') DEFAULT toDateTime(0),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4,\n INDEX idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS web_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 82957e5e2..418381e44 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "27015e7036e9cacaa5156bcc10a3aead96cb4fa2fcb7c615c272c691f2cbf54a" as const +export const projectRevision = "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c" as const export const datasources = [ { @@ -177,7 +177,7 @@ export const datasources = [ { name: "traces", content: - "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) `json:$.ResourceAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) `json:$.ScopeAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) `json:$.SpanAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes))\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) `json:$.ResourceAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) `json:$.ScopeAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) `json:$.SpanAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n AiVendor LowCardinality(String) `json:$.ai_vendor` DEFAULT '',\n AiSessionKeyState UInt8 `json:$.ai_session_key_state` DEFAULT 0,\n AiSessionKeyHash UInt64 `json:$.ai_session_key_hash` DEFAULT 0,\n AiRulesVersion UInt32 `json:$.ai_rules_version` DEFAULT 0,\n AiRollupHour DateTime('UTC') `json:$.ai_rollup_hour` DEFAULT toDateTime(0)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4\n idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4", }, { name: "traces_aggregates_hourly", diff --git a/packages/domain/src/tinybird/datasources.contract.test.ts b/packages/domain/src/tinybird/datasources.contract.test.ts index 11cd48566..7f06f10fc 100644 --- a/packages/domain/src/tinybird/datasources.contract.test.ts +++ b/packages/domain/src/tinybird/datasources.contract.test.ts @@ -69,6 +69,13 @@ const EXPECTED_TOPLEVEL_KEYS = { "links_span_id", "links_trace_state", "links_attributes", + // AI classification, emitted on every span by the ingest row builder. + // `ai_rollup_hour` is written whether or not classification is enabled. + "ai_vendor", + "ai_session_key_state", + "ai_session_key_hash", + "ai_rules_version", + "ai_rollup_hour", ]), metrics_sum: new Set([...metricCommonKeys(), "value", "aggregation_temporality", "is_monotonic"]), metrics_gauge: new Set([...metricCommonKeys(), "value"]), @@ -132,7 +139,7 @@ function topLevelKey(jsonPath: string): string | null { function emittedTopLevelKeys(datasource: DatasourceDefinition): Set { const keys = new Set() - for (const column of Object.values(datasource.options.schema)) { + for (const [name, column] of Object.entries(datasource.options.schema)) { const defaultExpression = ( column as { readonly type?: { @@ -140,9 +147,18 @@ function emittedTopLevelKeys(datasource: DatasourceDefinition): Set { } } ).type?.modifiers?.defaultExpression - if (defaultExpression !== undefined) continue const path = getColumnJsonPath(column) if (!path) continue + // The same rule `scripts/generate-clickhouse-insert-mappings.ts` applies: + // a column is treated as warehouse-computed — and so not something the + // gateway sends — only when it has a DEFAULT *expression* **and** an + // identity JSONPath (`$.`), which is what a column with no + // declared path falls back to. A DEFAULT expression paired with an + // explicitly declared path (`AiRollupHour` → `$.ai_rollup_hour`) is a + // column the gateway does emit, and keeps its DEFAULT only for rows + // written before it existed. + const isComputedIdentityPath = path === `$.${name}` || path === `$.${name}[:]` + if (defaultExpression !== undefined && isComputedIdentityPath) continue const top = topLevelKey(path) if (top) keys.add(top) } diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 048fc4b7f..f23c41c97 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -247,6 +247,36 @@ export const traces = defineDatasource("traces", { SpanAttributeItems: column(t.array(t.string()).defaultExpr(attributeItemsExpr("SpanAttributes")), { jsonPath: "$.SpanAttributeItems[:]", }), + /** + * AI classification verdict, written by the ingest classifier. + * + * Each column keeps its constant DEFAULT — rows written before the + * classifier shipped, and any writer that predates it, still read back + * correctly — but now also declares a **snake_case** JSONPath, matching + * the gateway-emitted columns above rather than the identity paths used + * by the computed-DEFAULT columns (`SampleRate`, `IsEntryPoint`, the + * `*AttributeItems`). That distinction is load-bearing: the insert-mapping + * generator drops a column when it has a DEFAULT *and* an identity + * JSONPath, on the assumption the gateway never emits it. These five are + * emitted on every span, so they must not match that shape. + */ + /** Normalized vendor slug from the closed allowlist. '' = not classified as AI. */ + AiVendor: column(t.string().lowCardinality().default(""), { jsonPath: "$.ai_vendor" }), + /** Frozen session-key quality enum, 0-6. Higher = stronger provenance. */ + AiSessionKeyState: column(t.uint8().default(0), { jsonPath: "$.ai_session_key_state" }), + /** cityHash64 of the session-key value. 0 unless AiSessionKeyState >= 5. */ + AiSessionKeyHash: column(t.uint64().default(0), { jsonPath: "$.ai_session_key_hash" }), + /** Classifier rule-set version. 0 = row predates classification; non-zero means + * the span was examined, including a non-AI verdict. */ + AiRulesVersion: column(t.uint32().default(0), { jsonPath: "$.ai_rules_version" }), + /** + * Receive-time-clamped rollup hour. Written unconditionally by the row + * builder for every span, classification flag on or off, as + * `YYYY-MM-DD HH:MM:SS` — no row may carry a garbage rollup hour. + */ + AiRollupHour: column(t.dateTime("UTC").defaultExpr("toDateTime(0)"), { + jsonPath: "$.ai_rollup_hour", + }), }, indexes: [ { @@ -291,6 +321,23 @@ export const traces = defineDatasource("traces", { type: "bloom_filter(0.01)", granularity: 1, }, + { + // set(0) is deliberately unbounded: the vendor domain is a closed + // ~30-value allowlist, and a capped set() that overflows silently + // degrades to always-match rather than erroring. + name: "idx_ai_vendor", + expr: "AiVendor", + type: "set(0)", + granularity: 4, + }, + { + // tokenbf rather than bloom_filter because registry scope matchers + // include prefix rules, which need token-level lookups. + name: "idx_scope_name", + expr: "ScopeName", + type: "tokenbf_v1(4096, 3, 0)", + granularity: 4, + }, ], engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index a084f9030..328c60c6a 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -16,6 +16,9 @@ import { LOCAL_SCHEMA_V4, LOCAL_SCHEMA_V4_MANIFEST_DIGEST, LOCAL_SCHEMA_V4_SQL, + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST_DIGEST, + LOCAL_SCHEMA_V5_SQL, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -104,6 +107,18 @@ if ( fail("the immutable local schema v4 snapshot no longer matches its historical identity") } +const v5 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V5.version) +if ( + !v5 || + LOCAL_SCHEMA_V5_MANIFEST_DIGEST !== v5.manifestDigest || + schemaFingerprint(LOCAL_SCHEMA_V5_SQL) !== v5.fingerprint || + schemaDigest(LOCAL_SCHEMA_V5_SQL) !== v5.digest || + LOCAL_SCHEMA_V5.fingerprint !== v5.fingerprint || + LOCAL_SCHEMA_V5.digest !== v5.digest +) { + fail("the immutable local schema v5 snapshot no longer matches its historical identity") +} + const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name) if (new Set(names).size !== names.length) fail("local structural schema manifest contains duplicate object names")