From b37aef03f839e50fa4e5d31b694527f60318d397 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 11 Aug 2026 22:30:17 +0200 Subject: [PATCH] feat(domain): add the hourly per-service AI vendor discovery rollup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `service_ai_vendors_hourly` and its materialized view (ClickHouse migration 0016, local-store migration v5 -> v6): one row per (org, service, vendor, hour) carrying span/eligibility/session-key-state counters plus uniqCombined(12) trace and session states, feeding read-path service pruning, vendor-per-service lookup, session-key health and sampling-exemption suggestions. The MV filters `AiVendor != ''`, so the HLL states never run on ordinary HTTP or DB traffic — that filter is both the cost model and the semantics, since post-enablement "no rows" then means "no AI spans". It writes synchronously inside the INSERT into `traces`, which makes this target's part-count and merge-lag trace-ingestion health rather than side-table health. There is no POPULATE and no backfill: correctness depends on the source rows having been classified, not on the view having existed, so the enablement hour is recorded instead and hours before it do not exist for readers. 0016 is `requiredForIngest: false` — it touches nothing the gateway inserts, so the ingest gate stays at 15. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 15 + .../ai-vendors-rollup.clickhouse.e2e.test.ts | 591 ++++++ apps/cli/src/server/local-schema-history.ts | 7 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 3 + .../v5-to-v6-service-ai-vendors-hourly.ts | 244 +++ apps/cli/src/server/schema-identity.ts | 17 +- apps/cli/src/server/schema/local-inserts.json | 2 +- .../cli/src/server/schema/local-schema-v6.sql | 1750 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 51 +- apps/cli/test/local-store-migrations.test.ts | 43 +- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- package.json | 2 +- packages/domain/src/ai-registry/index.ts | 5 + .../src/ai-registry/rollup-enablement.ts | 49 + .../0016_service_ai_vendors_hourly.ts | 89 + .../src/clickhouse/migrations/index.test.ts | 108 +- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 4 +- .../generated/tinybird-project-manifest.ts | 12 +- .../src/tinybird/ai-vendors-rollup-sql.ts | 34 + packages/domain/src/tinybird/datasources.ts | 119 ++ .../domain/src/tinybird/materializations.ts | 39 + .../src/tinybird/retention-matrix.test.ts | 7 +- scripts/check-local-schema-manifest.ts | 15 + 25 files changed, 3191 insertions(+), 21 deletions(-) create mode 100644 apps/api/src/services/warehouse/ai-vendors-rollup.clickhouse.e2e.test.ts create mode 100644 apps/cli/src/server/local-store-migrations/v5-to-v6-service-ai-vendors-hourly.ts create mode 100644 apps/cli/src/server/schema/local-schema-v6.sql create mode 100644 packages/domain/src/ai-registry/rollup-enablement.ts create mode 100644 packages/domain/src/clickhouse/migrations/0016_service_ai_vendors_hourly.ts create mode 100644 packages/domain/src/tinybird/ai-vendors-rollup-sql.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d32d399d..c0b69972d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -542,6 +542,21 @@ jobs: bun run --filter=@maple/api test -- src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts + # The AI vendor rollup's output is a coverage percentage shown to a + # customer, and every way it can be wrong renders fine: a broken + # counter identity, an HLL state merged at the wrong grouping, or a + # reader assuming merged parts all return plausible numbers. Only a + # real server over spans with known-by-construction answers catches it. + - name: Verify AI vendor rollup reader contract and parity + env: + CLICKHOUSE_E2E: "1" + CLICKHOUSE_E2E_URL: http://127.0.0.1:8123 + CLICKHOUSE_E2E_USER: maple + CLICKHOUSE_E2E_PASSWORD: maple + run: >- + bun run --filter=@maple/api test -- + src/services/warehouse/ai-vendors-rollup.clickhouse.e2e.test.ts + local-checkpoint-native: name: Local checkpoint native needs: changes diff --git a/apps/api/src/services/warehouse/ai-vendors-rollup.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/ai-vendors-rollup.clickhouse.e2e.test.ts new file mode 100644 index 000000000..9f4b0027e --- /dev/null +++ b/apps/api/src/services/warehouse/ai-vendors-rollup.clickhouse.e2e.test.ts @@ -0,0 +1,591 @@ +// Reader contract and raw-vs-rollup parity for `service_ai_vendors_hourly`. +// +// The rollup's whole value is a coverage ratio a customer is shown, and every +// way it can be wrong is silent: an HLL state that merges across the wrong +// grouping still returns a plausible percentage, a counter identity that stops +// holding still renders, and a reader that assumes one row per key returns the +// last-inserted part's numbers instead of the total. None of that is visible in +// SQL-text tests, so this suite runs the real migrations against a real server, +// seeds spans whose correct answers are known by construction, and checks the +// answers rather than the syntax. +// +// The failure modes it is shaped to catch: +// +// - The plan's CI identity breaking: +// `EligibleSpanCount = KeyAbsent + KeyInvalid + KeySubSession + KeySession`. +// A state enum that grows a value, or a `countIf` predicate that drifts, +// shows up here and nowhere else. +// - A reader assuming merged parts. The seed deliberately writes each key +// across several INSERTs so every key has multiple unmerged rows; a query +// without `GROUP BY` reads low, and `FINAL` is not an escape hatch. +// - Per-vendor coverage being mistaken for the headline. The two-vendor trace +// (litellm + langchain, key on langchain only) is the plan's worked example: +// merging TracesWithKey/TracesTotal across vendor rows gives 100%, while +// litellm's own row reads 0% on a fully resolvable trace. +// - `WeightedSpanCount` losing its floor. A span with SampleRate 0 must +// contribute 1.0, not 0 and not an infinity. +// - The MV grouping on something other than the stored `AiRollupHour` — a +// `toStartOfHour(Timestamp)` regression is invisible until a clock-skewed +// client opens a partition in 2038. +// - Non-AI traffic leaking into the rollup, which is both a cost regression +// and a semantics one: post-enablement, "no rows" must mean "no AI spans". + +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { + applyRealMigrations, + clickhouseE2eEnabled, + clickhouseExec, + uniqueDatabase, +} from "./clickhouse-e2e-support" + +const database = uniqueDatabase("maple_ai_vendors_rollup_e2e") + +const ORG_A = "org_ai_rollup_a" +const ORG_B = "org_ai_rollup_b" + +const HOUR_MS = 3_600_000 +const DAY_MS = 86_400_000 + +/** + * Anchored to *now*, not a fixed date. `traces` carries a 30-day TTL enforced at + * insert time, so a hardcoded calendar date silently drops every seeded row once + * it ages past the horizon — both sides of every comparison below go empty and + * the suite passes by comparing nothing to nothing. `the seed lands` exists to + * make that impossible. + */ +const BASE_MS = Math.floor((Date.now() - 3 * DAY_MS) / HOUR_MS) * HOUR_MS + +const chDateTime = (epochMs: number): string => new Date(epochMs).toISOString().replace("T", " ").slice(0, 19) + +/** Hour `n` of the fixture, as the value ingest would have written to AiRollupHour. */ +const hour = (n: number): string => chDateTime(BASE_MS + n * HOUR_MS) + +/** + * A seeded span. `sampleRate === undefined` means the column is left out of the + * INSERT entirely, so the table's DEFAULT expression computes it — the shape a + * writer that predates sample-awareness produces. + */ +interface SeedSpan { + readonly orgId: string + readonly service: string + readonly vendor: string + readonly traceId: string + readonly hourIndex: number + readonly state: number + readonly keyHash: string + readonly rulesVersion: number + readonly sampleRate?: number +} + +const span = ( + orgId: string, + service: string, + vendor: string, + traceId: string, + hourIndex: number, + state: number, + overrides: Partial = {}, +): SeedSpan => ({ + orgId, + service, + vendor, + traceId, + hourIndex, + state, + keyHash: state >= 5 ? "11" : "0", + rulesVersion: 7, + sampleRate: 1, + ...overrides, +}) + +/** + * A session-key hash above 2^53. UInt64 identity values corrupt as JS numbers, + * which is why the schema notes require `toString()` in the SELECT — this row + * makes a regression there a failing assertion rather than a rounding artifact + * nobody notices. + */ +const BIG_HASH = "9007199254740993" + +const SEED_SPANS: ReadonlyArray = [ + // ── The plan's worked case: one trace, two vendors, key on one of them. + // Merged across vendor rows the trace is covered; litellm's own row reads 0%. + span(ORG_A, "checkout", "langchain", "trace-mix", 0, 6, { keyHash: "101" }), + span(ORG_A, "checkout", "litellm", "trace-mix", 0, 3), + + // ── One trace spanning two hours: each hour counts it once, and merging the + // two hours must still yield one trace, not two. + span(ORG_A, "checkout", "openai", "trace-dup", 0, 6, { keyHash: "202" }), + span(ORG_A, "checkout", "openai", "trace-dup", 1, 6, { keyHash: "202" }), + + // ── Every state, so the eligibility identity is exercised rather than + // asserted over an all-state-6 population. 0/1/2 are AI spans that were never + // session-key eligible: they count in SpanCount but not EligibleSpanCount. + span(ORG_A, "checkout", "openai", "trace-s0", 0, 0), + span(ORG_A, "checkout", "openai", "trace-s1", 0, 1), + span(ORG_A, "checkout", "openai", "trace-s2", 0, 2), + span(ORG_A, "checkout", "openai", "trace-s3", 0, 3), + span(ORG_A, "checkout", "openai", "trace-s4", 0, 4), + span(ORG_A, "checkout", "openai", "trace-s5", 0, 5, { keyHash: "303" }), + span(ORG_A, "checkout", "openai", "trace-s6", 0, 6, { keyHash: BIG_HASH }), + + // ── SampleRate variants. 0 must floor to 1.0; the undefined one leaves the + // column out of the INSERT so the table's DEFAULT runs. + span(ORG_A, "worker", "openai", "trace-sr4", 0, 6, { sampleRate: 4, keyHash: "404" }), + span(ORG_A, "worker", "openai", "trace-sr0", 0, 6, { sampleRate: 0, keyHash: "505" }), + span(ORG_A, "worker", "openai", "trace-srd", 0, 3, { sampleRate: undefined }), + + // ── A second org and a second hour, so nothing can pass by accident on a + // single-group fixture and OrgId scoping is actually under test. + span(ORG_B, "checkout", "langchain", "trace-b1", 1, 6, { keyHash: "606", rulesVersion: 8 }), + span(ORG_B, "checkout", "langchain", "trace-b2", 1, 4, { rulesVersion: 9 }), + span(ORG_B, "worker", "litellm", "trace-b3", 2, 5, { keyHash: "707", rulesVersion: 8 }), +] + +/** + * Non-AI spans in the same orgs, services and hours. Post-enablement the rollup + * having no row for a service-hour must mean "genuinely no AI spans", so these + * must contribute nothing at all — not a zero row. + */ +const SEED_NON_AI: ReadonlyArray = [ + span(ORG_A, "checkout", "", "trace-plain-1", 0, 0), + span(ORG_A, "checkout", "", "trace-plain-2", 0, 0), + span(ORG_A, "worker", "", "trace-plain-3", 1, 0), + span(ORG_B, "checkout", "", "trace-plain-4", 1, 0), + // A non-AI span carrying a session-key state is the awkward one: if the MV + // ever filtered on state instead of vendor, this row would appear. + span(ORG_A, "checkout", "", "trace-plain-5", 0, 6, { keyHash: "808" }), +] + +/** + * Three INSERT batches over overlapping keys. Each lands its own part, so every + * rollup key ends up with several unmerged rows — which is what makes the + * "aggregate, never assume one row per key" assertions non-vacuous. The overlap + * also means duplicated *spans*: counters see them twice (correctly — they are + * distinct SpanIds only in name, and at-least-once counters are the documented + * behaviour of every Maple rollup), while `uniq` states do not. + */ +const AI_BATCHES: ReadonlyArray> = [ + SEED_SPANS.slice(0, 8), + SEED_SPANS.slice(8), + SEED_SPANS.slice(0, 4), +] + +const AI_SPANS_INSERTED = AI_BATCHES.reduce((total, batch) => total + batch.length, 0) + +const quote = (value: string): string => `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` + +const insertBatch = async (rows: ReadonlyArray): Promise => { + // Two column lists, because a row that wants the SampleRate DEFAULT must omit + // the column rather than pass a value. + const withRate = rows.filter((row) => row.sampleRate !== undefined) + const withoutRate = rows.filter((row) => row.sampleRate === undefined) + + const common = (row: SeedSpan): string => + `${quote(row.orgId)}, ${quote(chDateTime(BASE_MS + row.hourIndex * HOUR_MS + 60_000))}, ${quote(row.traceId)}, ${quote(`span-${row.traceId}-${row.vendor}-${row.hourIndex}`)}, 'op', 'Client', ${quote(row.service)}, 1000, 'Ok', ${quote(row.vendor)}, ${row.state}, ${row.keyHash}, ${row.rulesVersion}, ${quote(hour(row.hourIndex))}` + + if (withRate.length > 0) { + await clickhouseExec( + `INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, AiVendor, AiSessionKeyState, AiSessionKeyHash, AiRulesVersion, AiRollupHour, SampleRate) VALUES\n${withRate + .map((row) => `(${common(row)}, ${row.sampleRate})`) + .join(",\n")}`, + database, + ) + } + if (withoutRate.length > 0) { + await clickhouseExec( + `INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, AiVendor, AiSessionKeyState, AiSessionKeyHash, AiRulesVersion, AiRollupHour) VALUES\n${withoutRate + .map((row) => `(${common(row)})`) + .join(",\n")}`, + database, + ) + } +} + +const runJson = async (sql: string): Promise>> => { + const body = await clickhouseExec(sql, database, { + default_format: "JSON", + output_format_json_quote_64bit_integers: "0", + }) + const parsed = JSON.parse(body) as { readonly data?: ReadonlyArray> } + return parsed.data ?? [] +} + +const num = (value: unknown): number => Number(value) + +/** Stable, order-insensitive comparison of two result sets. */ +const canonical = (rows: ReadonlyArray>): string => + JSON.stringify([...rows].map((row) => JSON.stringify(row)).sort()) + +/** + * The reader contract, written out once: plain `sum`/`min`/`max` for the + * SimpleAggregateFunction columns, `uniqCombinedMerge(12)` for the states, an + * explicit `GROUP BY`, and no `FINAL` anywhere. + */ +const ROLLUP_READ_SQL = `SELECT + OrgId, + ServiceName, + AiVendor, + toString(Hour) AS Hour, + sum(SpanCount) AS SpanCount, + sum(WeightedSpanCount) AS WeightedSpanCount, + sum(EligibleSpanCount) AS EligibleSpanCount, + sum(KeyAbsentCount) AS KeyAbsentCount, + sum(KeyInvalidCount) AS KeyInvalidCount, + sum(KeySubSessionCount) AS KeySubSessionCount, + sum(KeySessionCount) AS KeySessionCount, + uniqCombinedMerge(12)(TracesTotal) AS TracesTotal, + uniqCombinedMerge(12)(TracesWithKey) AS TracesWithKey, + uniqCombinedMerge(12)(SessionsApprox) AS SessionsApprox, + min(RowRulesVersionMin) AS RowRulesVersionMin, + max(RowRulesVersionMax) AS RowRulesVersionMax, + max(RollupRulesVersion) AS RollupRulesVersion +FROM service_ai_vendors_hourly +GROUP BY OrgId, ServiceName, AiVendor, Hour +ORDER BY OrgId, ServiceName, AiVendor, Hour` + +/** + * The same numbers computed straight off `traces`. Identical expressions to the + * MV's, but evaluated at read time over the raw spans — so agreement means the + * MV's write-time evaluation and the state/merge round-trip preserved every + * value, and disagreement localizes to whichever column moved. + */ +const RAW_READ_SQL = `SELECT + OrgId, + ServiceName, + AiVendor, + toString(AiRollupHour) AS Hour, + count() AS SpanCount, + sum(if(SampleRate > 0, SampleRate, 1.0)) AS WeightedSpanCount, + countIf(AiSessionKeyState >= 3) AS EligibleSpanCount, + countIf(AiSessionKeyState = 3) AS KeyAbsentCount, + countIf(AiSessionKeyState = 4) AS KeyInvalidCount, + countIf(AiSessionKeyState = 5) AS KeySubSessionCount, + countIf(AiSessionKeyState = 6) AS KeySessionCount, + uniqCombined(12)(TraceId) AS TracesTotal, + uniqCombinedIf(12)(TraceId, AiSessionKeyState = 6) AS TracesWithKey, + uniqCombinedIf(12)(AiSessionKeyHash, AiSessionKeyState = 6) AS SessionsApprox, + min(AiRulesVersion) AS RowRulesVersionMin, + max(AiRulesVersion) AS RowRulesVersionMax, + max(AiRulesVersion) AS RollupRulesVersion +FROM traces +WHERE AiVendor != '' +GROUP BY OrgId, ServiceName, AiVendor, AiRollupHour +ORDER BY OrgId, ServiceName, AiVendor, Hour` + +describe.skipIf(!clickhouseE2eEnabled)("service_ai_vendors_hourly rollup", () => { + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE ${database}`) + await applyRealMigrations(database) + + // Three separate INSERTs over overlapping keys, on purpose: each one lands + // its own part, so every rollup key has several unmerged rows and a reader + // that skips the GROUP BY reads low instead of accidentally passing. + for (const batch of AI_BATCHES) await insertBatch(batch) + await insertBatch(SEED_NON_AI) + }, 180_000) + + afterAll(async () => { + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }, 30_000) + + it("creates the rollup table and its materialized view from the real migrations", async () => { + const recorded = await runJson("SELECT count() AS n FROM _maple_schema_migrations WHERE version = 16") + assert.strictEqual(num(recorded[0]?.n), 1, "migration 16 was not recorded") + + const objects = await runJson( + `SELECT name, engine + FROM system.tables + WHERE database = currentDatabase() + AND name IN ('service_ai_vendors_hourly', 'service_ai_vendors_hourly_mv') + ORDER BY name`, + ) + assert.deepStrictEqual( + objects.map((row) => [String(row.name), String(row.engine)]), + [ + ["service_ai_vendors_hourly", "AggregatingMergeTree"], + ["service_ai_vendors_hourly_mv", "MaterializedView"], + ], + ) + + // State types are not cast-compatible, so a widened or narrowed value type + // is a silent data loss rather than an error at write time. + const columns = await runJson( + `SELECT name, type + FROM system.columns + WHERE database = currentDatabase() + AND table = 'service_ai_vendors_hourly' + AND name IN ('OrgId', 'Hour', 'TracesTotal', 'TracesWithKey', 'SessionsApprox') + ORDER BY name`, + ) + assert.deepStrictEqual( + Object.fromEntries(columns.map((row) => [String(row.name), String(row.type)])), + { + Hour: "DateTime('UTC')", + // Must match traces.OrgId exactly. + OrgId: "LowCardinality(String)", + SessionsApprox: "AggregateFunction(uniqCombined(12), UInt64)", + TracesTotal: "AggregateFunction(uniqCombined(12), String)", + TracesWithKey: "AggregateFunction(uniqCombined(12), String)", + }, + ) + }) + + it("the seed lands, and lands as several unmerged parts per key", async () => { + // The guard against a vacuous suite: two empty result sets satisfy every + // equality below, and a TTL-expired seed produces exactly that. + const raw = await runJson("SELECT count() AS n FROM traces WHERE AiVendor != ''") + assert.strictEqual( + num(raw[0]?.n), + AI_SPANS_INSERTED, + "the AI span seed did not land — check the rows against the traces 30-day TTL", + ) + + // If merges have already collapsed everything, the no-FINAL assertions below + // would pass without proving anything. + const parts = await runJson( + `SELECT count() AS rows, uniqExact((OrgId, ServiceName, AiVendor, Hour)) AS keys + FROM service_ai_vendors_hourly`, + ) + assert.isAbove( + num(parts[0]?.rows), + num(parts[0]?.keys), + "every rollup key was already merged to one row — the multi-part reader assertions are vacuous", + ) + }) + + it("keeps the eligibility identity: Eligible = Absent + Invalid + SubSession + Session", async () => { + // The plan's CI assertion. Checked per group and in total, because a + // per-group break can cancel out in the total and vice versa. + const violations = await runJson( + `SELECT count() AS n FROM ( + ${ROLLUP_READ_SQL} + ) + WHERE EligibleSpanCount != KeyAbsentCount + KeyInvalidCount + KeySubSessionCount + KeySessionCount`, + ) + assert.strictEqual(num(violations[0]?.n), 0, "the eligibility identity broke for at least one group") + + const totals = await runJson( + `SELECT + sum(EligibleSpanCount) AS eligible, + sum(KeyAbsentCount + KeyInvalidCount + KeySubSessionCount + KeySessionCount) AS parts, + sum(SpanCount) AS spans + FROM service_ai_vendors_hourly`, + ) + assert.strictEqual(num(totals[0]?.eligible), num(totals[0]?.parts)) + // States 0-2 are AI spans that were never eligible; if this were an equality + // the fixture would have stopped covering the ineligible half of the enum. + assert.isBelow( + num(totals[0]?.eligible), + num(totals[0]?.spans), + "the fixture no longer contains ineligible AI spans", + ) + }) + + it("excludes non-AI spans entirely rather than writing zero rows", async () => { + const rollupSpans = await runJson("SELECT sum(SpanCount) AS n FROM service_ai_vendors_hourly") + const rawAiSpans = await runJson("SELECT count() AS n FROM traces WHERE AiVendor != ''") + assert.strictEqual(num(rollupSpans[0]?.n), num(rawAiSpans[0]?.n)) + + const emptyVendor = await runJson( + "SELECT count() AS n FROM service_ai_vendors_hourly WHERE AiVendor = ''", + ) + assert.strictEqual(num(emptyVendor[0]?.n), 0, "non-AI spans produced rollup rows") + + // The state-6 non-AI span in the fixture: it must not have contributed a + // session, which is what an MV filtering on state instead of vendor would do. + const sessions = await runJson( + `SELECT uniqCombinedMerge(12)(SessionsApprox) AS n + FROM service_ai_vendors_hourly`, + ) + const expectedSessions = new Set( + SEED_SPANS.filter((row) => row.state === 6).map((row) => row.keyHash), + ).size + assert.strictEqual( + num(sessions[0]?.n), + expectedSessions, + "a non-AI span's session key leaked into SessionsApprox", + ) + }) + + it("floors zero and defaulted SampleRate to 1.0 and keeps WeightedSpanCount finite", async () => { + const worker = await runJson( + `SELECT sum(WeightedSpanCount) AS weighted, sum(SpanCount) AS spans + FROM service_ai_vendors_hourly + WHERE OrgId = ${quote(ORG_A)} AND ServiceName = 'worker'`, + ) + // One span at 4.0, one at 0 (floors to 1.0), one with the column omitted so + // the table's DEFAULT computed it — which for an unsampled span is 1.0. + assert.strictEqual(num(worker[0]?.spans), 3) + assert.strictEqual(num(worker[0]?.weighted), 6) + + const finite = await runJson( + `SELECT count() AS n + FROM service_ai_vendors_hourly + WHERE NOT isFinite(WeightedSpanCount) OR WeightedSpanCount <= 0`, + ) + assert.strictEqual(num(finite[0]?.n), 0, "WeightedSpanCount went non-finite or non-positive") + }) + + it("groups on the stored AiRollupHour, not the span timestamp", async () => { + const hours = await runJson( + `SELECT DISTINCT toString(Hour) AS h FROM service_ai_vendors_hourly ORDER BY h`, + ) + assert.deepStrictEqual( + hours.map((row) => String(row.h)), + [hour(0), hour(1), hour(2)], + ) + + // Every seeded span's Timestamp sits a minute into its hour, so a + // toStartOfHour(Timestamp) regression would produce identical hours here. + // Pin the column instead: no rollup hour may disagree with the stored one. + const drift = await runJson( + `SELECT count() AS n FROM ( + SELECT DISTINCT AiRollupHour AS h FROM traces WHERE AiVendor != '' + ) AS raw + LEFT ANTI JOIN ( + SELECT DISTINCT Hour AS h FROM service_ai_vendors_hourly + ) AS rollup USING (h)`, + ) + assert.strictEqual(num(drift[0]?.n), 0, "a stored AiRollupHour has no matching rollup hour") + }) + + it("reads trace coverage by merging across vendor rows, not per vendor", async () => { + // The plan's worked example. `trace-mix` carries a langchain span with a + // session key and a litellm span without one. + const perVendor = await runJson( + `SELECT + AiVendor, + uniqCombinedMerge(12)(TracesTotal) AS total, + uniqCombinedMerge(12)(TracesWithKey) AS withKey + FROM service_ai_vendors_hourly + WHERE OrgId = ${quote(ORG_A)} AND ServiceName = 'checkout' AND Hour = ${quote(hour(0))} + AND AiVendor IN ('langchain', 'litellm') + GROUP BY AiVendor + ORDER BY AiVendor`, + ) + assert.deepStrictEqual( + perVendor.map((row) => [String(row.AiVendor), num(row.total), num(row.withKey)]), + [ + ["langchain", 1, 1], + // The diagnostic-only reading: 0% on a trace that is in fact fully + // resolvable. Asserting it pins *why* per-vendor ratios are not the + // headline, so nobody promotes them later. + ["litellm", 1, 0], + ], + ) + + // The headline: merge the same states across both vendor rows and the trace + // is covered exactly once. + const merged = await runJson( + `SELECT + uniqCombinedMerge(12)(TracesTotal) AS total, + uniqCombinedMerge(12)(TracesWithKey) AS withKey + FROM service_ai_vendors_hourly + WHERE OrgId = ${quote(ORG_A)} AND ServiceName = 'checkout' AND Hour = ${quote(hour(0))} + AND AiVendor IN ('langchain', 'litellm')`, + ) + assert.strictEqual(num(merged[0]?.total), 1) + assert.strictEqual(num(merged[0]?.withKey), 1) + }) + + it("counts a trace once per hour and once overall when it straddles hours", async () => { + // `trace-dup` appears in two hours. Each hour sees it; merging the hours + // must not double it — uniq merge is set union, which is the property the + // cross-shard note in the plan also depends on. + const perHour = await runJson( + `SELECT toString(Hour) AS h, uniqCombinedMerge(12)(TracesTotal) AS total + FROM service_ai_vendors_hourly + WHERE OrgId = ${quote(ORG_A)} AND ServiceName = 'checkout' AND AiVendor = 'openai' + GROUP BY Hour + ORDER BY Hour`, + ) + assert.deepStrictEqual( + perHour.map((row) => [String(row.h), num(row.total)]), + [ + // Hour 0 also carries the per-state fixture traces. + [hour(0), 8], + [hour(1), 1], + ], + ) + + const merged = await runJson( + `SELECT uniqCombinedMerge(12)(TracesTotal) AS total + FROM service_ai_vendors_hourly + WHERE OrgId = ${quote(ORG_A)} AND ServiceName = 'checkout' AND AiVendor = 'openai'`, + ) + assert.strictEqual(num(merged[0]?.total), 8, "the straddling trace was double-counted across hours") + }) + + it("carries a session-key hash above 2^53 without corrupting it", async () => { + const hashes = await runJson( + `SELECT toString(AiSessionKeyHash) AS h + FROM traces + WHERE AiVendor != '' AND TraceId = 'trace-s6' + LIMIT 1`, + ) + assert.strictEqual(String(hashes[0]?.h), BIG_HASH, "the >2^53 hash was corrupted on the raw table") + + // And it is a distinct session, not collapsed into a neighbour by a + // float round-trip through the HLL state. + const sessions = await runJson( + `SELECT uniqCombinedMerge(12)(SessionsApprox) AS n + FROM service_ai_vendors_hourly + WHERE OrgId = ${quote(ORG_A)} AND ServiceName = 'checkout' AND AiVendor = 'openai'`, + ) + // trace-dup's 202, trace-s6's big hash — trace-s5 is state 5, which does not + // feed SessionsApprox. + assert.strictEqual(num(sessions[0]?.n), 2) + }) + + it("reports rule versions as row provenance, with RollupRulesVersion equal by construction", async () => { + const versions = await runJson( + `SELECT + min(RowRulesVersionMin) AS lo, + max(RowRulesVersionMax) AS hi, + max(RollupRulesVersion) AS rollup + FROM service_ai_vendors_hourly + WHERE OrgId = ${quote(ORG_B)} AND ServiceName = 'checkout' AND AiVendor = 'langchain'`, + ) + // Two spans written by v8 and v9 in the same group: the range is visible + // rather than collapsed to one number. + assert.strictEqual(num(versions[0]?.lo), 8) + assert.strictEqual(num(versions[0]?.hi), 9) + // For MV-written rows this always equals RowRulesVersionMax; divergence is + // the signal that a partition was rebuilt by a later registry version. + assert.strictEqual(num(versions[0]?.rollup), num(versions[0]?.hi)) + }) + + it("agrees with the same aggregates computed straight off traces", async () => { + const [rollupRows, rawRows] = await Promise.all([runJson(ROLLUP_READ_SQL), runJson(RAW_READ_SQL)]) + + assert.isNotEmpty(rawRows, "the parity fixture produced no raw groups") + assert.strictEqual( + rollupRows.length, + rawRows.length, + "the rollup and the raw table disagree on how many (org, service, vendor, hour) groups exist", + ) + assert.strictEqual(canonical(rollupRows), canonical(rawRows)) + }) + + it("does not need FINAL, and a reader that skips the GROUP BY reads low", async () => { + // Stated as a property rather than a style rule: the contract is that + // aggregating without FINAL is correct, and that the unaggregated read is + // wrong — which is what makes the GROUP BY mandatory rather than decorative. + const aggregated = await runJson("SELECT sum(SpanCount) AS n FROM service_ai_vendors_hourly") + const naive = await runJson( + `SELECT SpanCount AS n + FROM service_ai_vendors_hourly + ORDER BY SpanCount DESC + LIMIT 1`, + ) + assert.isAbove( + num(aggregated[0]?.n), + num(naive[0]?.n), + "a single unaggregated row already carried the total — the fixture stopped exercising multiple parts", + ) + + // FINAL must be unnecessary, not merely discouraged: same answer either way. + const withFinal = await runJson("SELECT sum(SpanCount) AS n FROM service_ai_vendors_hourly FINAL") + assert.strictEqual(num(aggregated[0]?.n), num(withFinal[0]?.n)) + }) +}) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 48c77f2ad..a2f94123f 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -58,4 +58,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "e0b0e0a9af30cc7aca51cec02c566dab9f4cbfda1374c177a7caee9a46a31783", projectRevision: "09513d18e8cdea657efa56dbe764defebe66a28e5397411dc03fadb7f19f1c58", }), + Object.freeze({ + version: 6, + fingerprint: "3237cdb572f16c18", + digest: "3237cdb572f16c18c516971c91373402bf5d00bcf05a0943c15d517ad4ec58d4", + manifestDigest: "93b7ea569739e51153c21fa0383de803656861e7bd074b69847323c53c9afb86", + projectRevision: "097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 40530abf3..9ea99d34f 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 5 as const +export const LOCAL_SCHEMA_VERSION = 6 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index cecf0af6e..a1c3ee5ef 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -37,6 +37,7 @@ import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" import { v4ToV5AiClassificationColumnsModule } from "./local-store-migrations/v4-to-v5-ai-classification-columns" +import { v5ToV6ServiceAiVendorsHourlyModule } from "./local-store-migrations/v5-to-v6-service-ai-vendors-hourly" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -60,6 +61,7 @@ export { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error export { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" export { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" export { v4ToV5AiClassificationColumnsModule } from "./local-store-migrations/v4-to-v5-ai-classification-columns" +export { v5ToV6ServiceAiVendorsHourlyModule } from "./local-store-migrations/v5-to-v6-service-ai-vendors-hourly" const NONTERMINAL_PHASES = new Set([ "planned", @@ -125,6 +127,7 @@ export const localStoreMigrations: ReadonlyArray = v2ToV3ServiceMapIngestBridgeModule, v3ToV4WebEventsModule, v4ToV5AiClassificationColumnsModule, + v5ToV6ServiceAiVendorsHourlyModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v5-to-v6-service-ai-vendors-hourly.ts b/apps/cli/src/server/local-store-migrations/v5-to-v6-service-ai-vendors-hourly.ts new file mode 100644 index 000000000..9501638d2 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v5-to-v6-service-ai-vendors-hourly.ts @@ -0,0 +1,244 @@ +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V5_SQL, + LOCAL_SCHEMA_V6, + LOCAL_SCHEMA_V6_MANIFEST, + LOCAL_SCHEMA_V6_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +interface V5ToV6State { + readonly module: "local-0005-to-0006-service-ai-vendors-hourly" + readonly version: 1 + readonly rawRows: Readonly> + readonly retentionDays?: number +} + +interface V5ToV6Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v5 -> v6 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (typeof count !== "string" || !/^\d+$/.test(count)) + throw new Error(`v5 -> v6 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v5 -> v6 rawRows contains an unknown table") + return counts +} + +const decodeState = (value: unknown): V5ToV6State => { + if (!isRecord(value)) throw new Error("v5 -> v6 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v5 -> v6 state contains an unknown field") + if (value.module !== "local-0005-to-0006-service-ai-vendors-hourly" || value.version !== 1) + throw new Error("v5 -> v6 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v5 -> v6 retentionDays must be an integer") + return { + module: "local-0005-to-0006-service-ai-vendors-hourly", + version: 1, + rawRows: decodeCounts(value.rawRows), + ...(value.retentionDays === undefined ? {} : { retentionDays: value.retentionDays }), + } +} + +const decodeProgress = (value: unknown): V5ToV6Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v5 -> v6 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V5_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V5_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, + ) + return { + module: "local-0005-to-0006-service-ai-vendors-hourly", + version: 1, + rawRows, + ...(retentionDays === undefined ? {} : { retentionDays }), + } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V5ToV6State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Purely additive: one new table and the view that fills it. Nothing existing is + * touched, so — like v3 -> v4 and unlike v4 -> v5 — bootstrapping the v6 DDL is + * the whole migration. Every other object's `CREATE … IF NOT EXISTS` is a no-op + * against the cloned store; only `service_ai_vendors_hourly` and its MV are new. + * There is deliberately no explicit ALTER list here: the objects are *created*, + * not modified, so the generated DDL is already the single source of truth and a + * hand-copied CREATE would be a second one. + * + * `service_ai_vendors_hourly` starts empty and stays that way for the store's + * existing history — a materialized view is an insert trigger, so it only sees + * spans written after this point. That is the same position a deployed cluster + * is in after ClickHouse migration 0016, and it is why that migration ships no + * POPULATE either. Backfilling here would mean rewriting a store we have just + * promised to clone byte-for-byte, and it would be wrong on top of that: rows + * written before the ingest classifier ran carry `AiVendor = ''`, so a backfill + * would produce not a partial rollup but an empty one, indistinguishable from + * "this store genuinely has no AI spans". + */ +const apply = async (context: MigrationModuleContext): Promise => + context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V6_SQL, + bootstrapSchema: true, + }) + +const verify = async ( + context: MigrationModuleContext, + state: V5ToV6State, + _progress: V5ToV6Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V6_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v5 -> v6 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V6_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v5-store", + description: "Clone the stopped v5 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "install-service-ai-vendors-hourly", + description: "Install the AI vendor discovery rollup and its materialized view", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v6-schema", + description: "Verify the v6 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v5 store is cloned byte-for-byte before additive DDL runs.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "The source of the new view is neither read nor rewritten; the rollup fills from spans written after the migration.", + }, + { + // Created empty and filled forward, never backfilled — twice over. The + // store was just promised byte-for-byte, and the pre-migration rows would + // not produce a usable rollup anyway: local mode has no ingest classifier, + // so every existing span carries AiVendor = '' and the view's WHERE + // excludes all of them. Unlike web_events, this table does NOT converge + // with its source's horizon — it keeps 400 days against traces' 30 — so + // what converges is the *overlap*: past 30 days the raw spans are gone and + // the rollup is the only record, complete from the migration forward. + name: "service_ai_vendors_hourly", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Filled forward from classified spans written after the migration; the raw source retains 30 days, so nothing older than that horizon was ever rebuildable from this store.", + preservationInterval: "traces retention horizon", + sourceRetentionDays: 30, + targetRetentionDays: 400, + }, +] + +export const v5ToV6ServiceAiVendorsHourlyModule: LocalStoreMigrationModule = { + id: "local-0005-to-0006-service-ai-vendors-hourly", + moduleVersion: 1, + description: "Add the AI vendor discovery rollup and its materialized view to v5", + from: LOCAL_SCHEMA_V5, + to: LOCAL_SCHEMA_V6, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 4cfe6f033..6183958a7 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -4,6 +4,7 @@ import schemaV2Sql from "./schema/local-schema-v2.sql" with { type: "text" } import schemaV3Sql from "./schema/local-schema-v3.sql" with { type: "text" } import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } +import schemaV6Sql from "./schema/local-schema-v6.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -27,7 +28,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "09513d18e8cdea657efa56dbe764defebe66a28e5397411dc03fadb7f19f1c58" + "097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -63,6 +64,11 @@ export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest +/** Immutable v6 DDL/manifest snapshot used by the v5 -> v6 module after the + * generated current schema advances. */ +export const LOCAL_SCHEMA_V6_SQL = schemaV6Sql +export const LOCAL_SCHEMA_V6_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV6Sql) +export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = LOCAL_SCHEMA_V6_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -122,6 +128,15 @@ export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, }) +export const LOCAL_SCHEMA_V6: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_HISTORY[6]!.version, + fingerprint: LOCAL_SCHEMA_HISTORY[6]!.fingerprint, + digest: LOCAL_SCHEMA_HISTORY[6]!.digest, + manifestDigest: LOCAL_SCHEMA_HISTORY[6]!.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: LOCAL_SCHEMA_HISTORY[6]!.projectRevision, +}) + export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, fingerprint: SCHEMA_FINGERPRINT, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 4244958b6..c3840d2af 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c", + "projectRevision": "097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v6.sql b/apps/cli/src/server/schema/local-schema-v6.sql new file mode 100644 index 000000000..281c7207a --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v6.sql @@ -0,0 +1,1750 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7 +-- localSchemaVersion: 6 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_ai_vendors_hourly ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + AiVendor LowCardinality(String), + Hour DateTime('UTC'), + SpanCount SimpleAggregateFunction(sum, UInt64), + WeightedSpanCount SimpleAggregateFunction(sum, Float64), + EligibleSpanCount SimpleAggregateFunction(sum, UInt64), + KeyAbsentCount SimpleAggregateFunction(sum, UInt64), + KeyInvalidCount SimpleAggregateFunction(sum, UInt64), + KeySubSessionCount SimpleAggregateFunction(sum, UInt64), + KeySessionCount SimpleAggregateFunction(sum, UInt64), + TracesTotal AggregateFunction(uniqCombined(12), String), + TracesWithKey AggregateFunction(uniqCombined(12), String), + SessionsApprox AggregateFunction(uniqCombined(12), UInt64), + RowRulesVersionMin SimpleAggregateFunction(min, UInt32), + RowRulesVersionMax SimpleAggregateFunction(max, UInt32), + RollupRulesVersion SimpleAggregateFunction(max, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMMDD(Hour) +ORDER BY (OrgId, ServiceName, AiVendor, Hour) +TTL Hour + INTERVAL 400 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + AiVendor LowCardinality(String) DEFAULT '', + AiSessionKeyState UInt8 DEFAULT 0, + AiSessionKeyHash UInt64 DEFAULT 0, + AiRulesVersion UInt32 DEFAULT 0, + AiRollupHour DateTime('UTC') DEFAULT toDateTime(0), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4, + INDEX idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_ai_vendors_hourly_mv TO service_ai_vendors_hourly AS +SELECT + OrgId, + ServiceName, + AiVendor, + AiRollupHour AS Hour, + count() AS SpanCount, + sum(if(SampleRate > 0, SampleRate, 1.0)) AS WeightedSpanCount, + countIf(AiSessionKeyState >= 3) AS EligibleSpanCount, + countIf(AiSessionKeyState = 3) AS KeyAbsentCount, + countIf(AiSessionKeyState = 4) AS KeyInvalidCount, + countIf(AiSessionKeyState = 5) AS KeySubSessionCount, + countIf(AiSessionKeyState = 6) AS KeySessionCount, + uniqCombinedState(12)(TraceId) AS TracesTotal, + uniqCombinedStateIf(12)(TraceId, AiSessionKeyState = 6) AS TracesWithKey, + uniqCombinedStateIf(12)(AiSessionKeyHash, AiSessionKeyState = 6) AS SessionsApprox, + min(AiRulesVersion) AS RowRulesVersionMin, + max(AiRulesVersion) AS RowRulesVersionMax, + max(AiRulesVersion) AS RollupRulesVersion +FROM traces +WHERE AiVendor != '' +GROUP BY OrgId, ServiceName, AiVendor, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 9c147d9ad..281c7207a 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c --- localSchemaVersion: 5 +-- projectRevision: 097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7 +-- localSchemaVersion: 6 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -345,6 +345,30 @@ PARTITION BY toDate(Hour) ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) TTL toDate(Hour) + INTERVAL 365 DAY; +CREATE TABLE IF NOT EXISTS service_ai_vendors_hourly ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + AiVendor LowCardinality(String), + Hour DateTime('UTC'), + SpanCount SimpleAggregateFunction(sum, UInt64), + WeightedSpanCount SimpleAggregateFunction(sum, Float64), + EligibleSpanCount SimpleAggregateFunction(sum, UInt64), + KeyAbsentCount SimpleAggregateFunction(sum, UInt64), + KeyInvalidCount SimpleAggregateFunction(sum, UInt64), + KeySubSessionCount SimpleAggregateFunction(sum, UInt64), + KeySessionCount SimpleAggregateFunction(sum, UInt64), + TracesTotal AggregateFunction(uniqCombined(12), String), + TracesWithKey AggregateFunction(uniqCombined(12), String), + SessionsApprox AggregateFunction(uniqCombined(12), UInt64), + RowRulesVersionMin SimpleAggregateFunction(min, UInt32), + RowRulesVersionMax SimpleAggregateFunction(max, UInt32), + RollupRulesVersion SimpleAggregateFunction(max, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMMDD(Hour) +ORDER BY (OrgId, ServiceName, AiVendor, Hour) +TTL Hour + INTERVAL 400 DAY; + CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( OrgId LowCardinality(String), Hour DateTime, @@ -1177,6 +1201,29 @@ SELECT FROM metrics_sum GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; +CREATE MATERIALIZED VIEW IF NOT EXISTS service_ai_vendors_hourly_mv TO service_ai_vendors_hourly AS +SELECT + OrgId, + ServiceName, + AiVendor, + AiRollupHour AS Hour, + count() AS SpanCount, + sum(if(SampleRate > 0, SampleRate, 1.0)) AS WeightedSpanCount, + countIf(AiSessionKeyState >= 3) AS EligibleSpanCount, + countIf(AiSessionKeyState = 3) AS KeyAbsentCount, + countIf(AiSessionKeyState = 4) AS KeyInvalidCount, + countIf(AiSessionKeyState = 5) AS KeySubSessionCount, + countIf(AiSessionKeyState = 6) AS KeySessionCount, + uniqCombinedState(12)(TraceId) AS TracesTotal, + uniqCombinedStateIf(12)(TraceId, AiSessionKeyState = 6) AS TracesWithKey, + uniqCombinedStateIf(12)(AiSessionKeyHash, AiSessionKeyState = 6) AS SessionsApprox, + min(AiRulesVersion) AS RowRulesVersionMin, + max(AiRulesVersion) AS RowRulesVersionMax, + max(AiRulesVersion) AS RollupRulesVersion +FROM traces +WHERE AiVendor != '' +GROUP BY OrgId, ServiceName, AiVendor, Hour; + CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS SELECT OrgId, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 16c07b3e8..7f07b686b 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -15,6 +15,7 @@ import { LOCAL_SCHEMA_V4_MANIFEST, LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V6, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -56,16 +57,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v5 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("b3059dd34e85858f") - expect(SCHEMA_DIGEST).toBe("b3059dd34e85858f8893cd7fc88d9c28f489992c39fb2a334f8caf1747a69c21") + it("matches the generated v6 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("3237cdb572f16c18") + expect(SCHEMA_DIGEST).toBe("3237cdb572f16c18c516971c91373402bf5d00bcf05a0943c15d517ad4ec58d4") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(5) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V5) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(6) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V6) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -130,6 +131,30 @@ describe("current local schema identity", () => { "idx_ai_vendor", "idx_scope_name", ]) + + // v6 is exactly v5 plus the AI vendor discovery rollup and its view — no + // column touched anywhere else. Asserted as a name delta against the frozen + // v5 manifest so a stray table can't ride along on this version. + const v5Names = new Set(LOCAL_SCHEMA_V5_MANIFEST.objects.map((object) => object.name)) + expect( + LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name).filter((name) => !v5Names.has(name)), + ).toEqual(["service_ai_vendors_hourly", "service_ai_vendors_hourly_mv"]) + const aiVendors = LOCAL_SCHEMA_MANIFEST.objects.find( + (object) => object.name === "service_ai_vendors_hourly", + ) + expect(aiVendors?.engine).toBe("AggregatingMergeTree") + expect(aiVendors?.orderBy).toBe("(OrgId, ServiceName, AiVendor, Hour)") + // Daily partitions and a 400-day TTL against a 30-day source: the deliberate + // retention asymmetry, and the reason a rebuild can only repair closed days + // whose raw spans still exist. + expect(aiVendors?.partitionBy).toBe("toYYYYMMDD(Hour)") + expect(aiVendors?.ttl).toBe("Hour + INTERVAL 400 DAY") + const aiVendorsView = LOCAL_SCHEMA_MANIFEST.objects.find( + (object) => object.name === "service_ai_vendors_hourly_mv", + ) + expect(aiVendorsView?.definition).toContain("FROM traces") + // Without the filter, every span in a local store would enter the HLL states. + expect(aiVendorsView?.definition).toContain("WHERE AiVendor != ''") }) }) @@ -142,6 +167,7 @@ describe("local migration registry", () => { "local-0002-to-0003-service-map-ingest-bridge", "local-0003-to-0004-web-events", "local-0004-to-0005-ai-classification-columns", + "local-0005-to-0006-service-ai-vendors-hourly", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -149,6 +175,7 @@ describe("local migration registry", () => { expect(chain[2]?.to).toEqual(LOCAL_SCHEMA_V3) expect(chain[3]?.to).toEqual(LOCAL_SCHEMA_V4) expect(chain[4]?.to).toEqual(LOCAL_SCHEMA_V5) + expect(chain[5]?.to).toEqual(LOCAL_SCHEMA_V6) expect(typeof chain[0]?.apply).toBe("function") }) @@ -170,12 +197,12 @@ describe("local migration registry", () => { maple: "dev", createdAt: "2026-01-01T00:00:00.000Z", createdByMaple: "dev", - schemaVersion: 5, + schemaVersion: 6, schemaDigest: SCHEMA_DIGEST, schema: SCHEMA_FINGERPRINT, activation: "active", }), - ).toMatchObject({ version: 5, fingerprint: SCHEMA_FINGERPRINT, digest: SCHEMA_DIGEST }) + ).toMatchObject({ version: 6, fingerprint: SCHEMA_FINGERPRINT, digest: SCHEMA_DIGEST }) }) it("rejects unknown, future, downgrade, and ambiguous paths", () => { @@ -184,7 +211,7 @@ describe("local migration registry", () => { ).toThrow(/no registered/) expect(() => resolveMigrationChain( - { ...CURRENT_LOCAL_SCHEMA, version: 6, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 7, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index bb91d281c..5673ad7dc 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c"; +pub const PROJECT_REVISION: &str = "097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/package.json b/package.json index 0647994d0..7e0eda2af 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "db:up": "docker compose up -d postgres electric", "ch:up": "docker compose -f docker-compose.development.yml up -d clickhouse", "ch:down": "docker compose -f docker-compose.development.yml stop clickhouse", - "ch:test": "CLICKHOUSE_E2E=1 CLICKHOUSE_E2E_URL=http://127.0.0.1:8123 bun run --filter=@maple/api test -- src/services/warehouse/sql-catalog.clickhouse.e2e.test.ts src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts", + "ch:test": "CLICKHOUSE_E2E=1 CLICKHOUSE_E2E_URL=http://127.0.0.1:8123 bun run --filter=@maple/api test -- src/services/warehouse/sql-catalog.clickhouse.e2e.test.ts src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts src/services/warehouse/ai-vendors-rollup.clickhouse.e2e.test.ts", "db:migrate:local": "DATABASE_URL=postgres://maple:maple@localhost:5499/maple bun run --cwd packages/db db:migrate", "migrate:prod": "bun run --cwd packages/db ps:apply-schema main", "dev:alerting": "bun --filter=@maple/alerting dev", diff --git a/packages/domain/src/ai-registry/index.ts b/packages/domain/src/ai-registry/index.ts index 6923a87f9..19837558a 100644 --- a/packages/domain/src/ai-registry/index.ts +++ b/packages/domain/src/ai-registry/index.ts @@ -31,6 +31,11 @@ export { type Variant, } from "./schema" +export { + AI_VENDORS_ROLLUP_ENABLEMENT_HOUR_ENV, + AI_VENDORS_ROLLUP_TABLE, +} from "./rollup-enablement" + export { PRIORITY_BANDS, UNKNOWN_VENDOR_PREFIX, diff --git a/packages/domain/src/ai-registry/rollup-enablement.ts b/packages/domain/src/ai-registry/rollup-enablement.ts new file mode 100644 index 000000000..d82d75f06 --- /dev/null +++ b/packages/domain/src/ai-registry/rollup-enablement.ts @@ -0,0 +1,49 @@ +/** + * Reserved configuration name for the `service_ai_vendors_hourly` enablement + * hour. v1 reserves the name and states the reader rule; nothing consumes it + * yet, because the read path does not exist yet. + * + * ## Why a boundary is needed at all + * + * The rollup's materialized view is an insert trigger: it only ever sees spans + * written after it was created. The hour in which it is created is therefore + * **truncated but indistinguishable from a complete hour** — every span the + * classifier examined was classified correctly, the counters are internally + * consistent, `EligibleSpanCount` still equals the sum of the four state + * counters, and nothing in the row says "this hour started before I existed". + * A coverage ratio computed over it is not wrong so much as computed over an + * arbitrary suffix of the hour. + * + * The design that would have made this self-marking — an all-traffic watermark + * table whose version-0 rows exposed partial-classification windows — was + * deliberately dropped: production classifies unconditionally, so partial + * windows are migration-scoped rather than ongoing, and paying for a + * second all-traffic MV to mark a one-time boundary was the wrong trade. This + * config value is what replaced it. + * + * ## The reader rule + * + * Readers must treat every hour **strictly before** this value as nonexistent — + * not as zero. "No AI spans this hour" and "the rollup was not recording this + * hour" are different claims, and only the first one may be shown to a customer. + * A query window that starts before the boundary is clamped forward to it, and + * the UI says so rather than silently narrowing. + * + * A rollback and re-enable during the migration window produces a *new* + * boundary; the hours between the two are unclassified and fall under the same + * rule. Post-rollout there is no flag, so the boundary never moves again. + * + * ## Format + * + * An ISO-8601 UTC hour with no sub-hour component — `2026-08-12T00:00:00Z` — + * naming the first hour the rollup covers in full. It is set by the deployment + * that creates the MV (ClickHouse migration 0016), to the first hour boundary + * strictly after MV creation. + */ +export const AI_VENDORS_ROLLUP_ENABLEMENT_HOUR_ENV = "AI_VENDORS_ROLLUP_ENABLEMENT_HOUR" as const + +/** + * Name of the rollup the boundary above governs. Kept beside it so a grep for + * either one finds the other. + */ +export const AI_VENDORS_ROLLUP_TABLE = "service_ai_vendors_hourly" as const diff --git a/packages/domain/src/clickhouse/migrations/0016_service_ai_vendors_hourly.ts b/packages/domain/src/clickhouse/migrations/0016_service_ai_vendors_hourly.ts new file mode 100644 index 000000000..fd61074d4 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0016_service_ai_vendors_hourly.ts @@ -0,0 +1,89 @@ +import { SERVICE_AI_VENDORS_HOURLY_SELECT_SQL } from "../../tinybird/ai-vendors-rollup-sql" + +/** + * Migration 0016 — the AI vendor discovery rollup and its live-write MV. + * + * Installs the empty target and the view that fills it forward. Deliberately no + * `POPULATE` and no backfill spec: unlike 0008/0014 there is nothing safe to + * backfill here, because correctness depends on the *source rows* having been + * classified, not on the view having existed. + * + * ## Deploy gate (runbook precondition, not code) + * + * The ingest classification flag must be at **100% for at least one full clock + * hour** before this migration runs in production. The write-side plan's §8 gate + * is hard, not a preference: the watermark table that would have made partially + * classified hours self-marking was removed, so a partial hour is + * indistinguishable from a complete one at read time. Rolling this out mid-ramp + * produces an hour of rows that look healthy and undercount. + * + * Rollback is migration-window-only (production has no flag): flag off ⇒ spans + * arrive with `AiVendor = ''`, `AiRulesVersion = 0`, and the rollup simply stops + * accreting because the MV's `WHERE` no longer matches. Re-enabling records a + * *fresh* enablement boundary; readers treat the hours between the two + * boundaries as unclassified. + * + * ## Enablement hour + * + * The hour this view is created is truncated — the classifier saw every span, + * but the view did not exist for part of the hour — and nothing about the stored + * row distinguishes it from a complete hour. The deployment therefore records + * the first fully covered hour in config + * (`AI_VENDORS_ROLLUP_ENABLEMENT_HOUR_ENV`, see `../../ai-registry`) and readers + * treat everything before it as nonexistent. v1 reserves the name and documents + * the rule; no consumer reads it yet because no read path exists yet. + * + * ## Org deletion and retention + * + * `ALTER TABLE service_ai_vendors_hourly DELETE WHERE OrgId = '…'` is the one + * sanctioned mutation on this table — well-pruned, since `OrgId` is the + * sort-key prefix. It is the named exception to the no-mutation rule the rest of + * the warehouse follows. + * + * Retention is deliberately asymmetric: 400 days here against `traces`' 30. The + * rollup outlives the raw spans it was derived from, which is the point (a year + * of vendor history costs a few thousand rows per org) and also the constraint — + * past the raw horizon these rows cannot be rebuilt from anything, so a registry + * fix can only repair partitions whose source spans still exist. + * + * `requiredForIngest: false`: the gateway's INSERT column list is untouched, so a + * BYO cluster still at 15 keeps ingesting correctly and only misses a read-path + * concern. `clickHouseSchemaVersion` stays "15" for exactly that reason — + * un-readying every BYO org over a rollup they do not yet query would route their + * ingest to managed Tinybird while the dashboard kept reading their cluster. + * + * The `CREATE MATERIALIZED VIEW` body is imported, not retyped: the deployed view + * and the migrated view must be the same query, and the index test asserts it. + */ +export const migration_0016_service_ai_vendors_hourly = { + version: 16, + description: "Add the hourly AI vendor discovery rollup and its live-write materialized view", + requiredForIngest: false, + statements: [ + `CREATE TABLE IF NOT EXISTS service_ai_vendors_hourly ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + AiVendor LowCardinality(String), + Hour DateTime('UTC'), + SpanCount SimpleAggregateFunction(sum, UInt64), + WeightedSpanCount SimpleAggregateFunction(sum, Float64), + EligibleSpanCount SimpleAggregateFunction(sum, UInt64), + KeyAbsentCount SimpleAggregateFunction(sum, UInt64), + KeyInvalidCount SimpleAggregateFunction(sum, UInt64), + KeySubSessionCount SimpleAggregateFunction(sum, UInt64), + KeySessionCount SimpleAggregateFunction(sum, UInt64), + TracesTotal AggregateFunction(uniqCombined(12), String), + TracesWithKey AggregateFunction(uniqCombined(12), String), + SessionsApprox AggregateFunction(uniqCombined(12), UInt64), + RowRulesVersionMin SimpleAggregateFunction(min, UInt32), + RowRulesVersionMax SimpleAggregateFunction(max, UInt32), + RollupRulesVersion SimpleAggregateFunction(max, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMMDD(Hour) +ORDER BY (OrgId, ServiceName, AiVendor, Hour) +TTL Hour + INTERVAL 400 DAY`, + `CREATE MATERIALIZED VIEW IF NOT EXISTS service_ai_vendors_hourly_mv TO service_ai_vendors_hourly AS +${SERVICE_AI_VENDORS_HOURLY_SELECT_SQL}`, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 7444d93b7..5a56c93b9 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -18,6 +18,9 @@ import { migration_0012_session_event_attribute_keys } from "./0012_session_even import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events, webEventsBackfill } from "./0014_web_events" import { migration_0015_ai_classification_columns } from "./0015_ai_classification_columns" +import { migration_0016_service_ai_vendors_hourly } from "./0016_service_ai_vendors_hourly" +import { SERVICE_AI_VENDORS_HOURLY_SELECT_SQL } from "../../tinybird/ai-vendors-rollup-sql" +import { latestSnapshotStatements } from "../../generated/clickhouse-schema" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -33,10 +36,10 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ]) - expect(migrations.at(-1)).toBe(migration_0015_ai_classification_columns) - expect(latestMigrationVersion).toBe(15) + expect(migrations.at(-1)).toBe(migration_0016_service_ai_vendors_hourly) + expect(latestMigrationVersion).toBe(16) // 0010 and 0014 are performance/storage-only, so the ingest-gating version // skips both — nothing writes `web_events` directly and search indexes // change nothing the gateway sends, and bumping for either would un-ready @@ -50,9 +53,108 @@ describe("ClickHouse migrations", () => { // `clickhouse_ready = false` and routes to the managed pipeline until its // schema syncs. expect(migration_0015_ai_classification_columns.requiredForIngest).toBe(true) + + // 0016 adds a read-path rollup and touches nothing the gateway inserts, so + // the ingest gate stays at 15. Bumping it would un-ready every BYO-CH org + // over a table their ingest path never writes. + expect(migration_0016_service_ai_vendors_hourly.requiredForIngest).toBe(false) expect(clickHouseSchemaVersion).toBe("15") }) + it("installs the AI vendor rollup and its live-write MV with no POPULATE and no backfill", () => { + const sql = migration_0016_service_ai_vendors_hourly.statements.join("\n\n") + + expect(sql).toContain("CREATE TABLE IF NOT EXISTS service_ai_vendors_hourly") + expect(sql).toContain("ENGINE = AggregatingMergeTree") + // Daily partitions: a registry-fix rebuild is a per-closed-day atomic + // REPLACE PARTITION, which monthly partitions would make unaffordable. + expect(sql).toContain("PARTITION BY toYYYYMMDD(Hour)") + // OrgId first so the one sanctioned mutation (org deletion) prunes. + expect(sql).toContain("ORDER BY (OrgId, ServiceName, AiVendor, Hour)") + // 400 days against a 30-day source — deliberate asymmetry, see the migration doc. + expect(sql).toContain("TTL Hour + INTERVAL 400 DAY") + + // State types are not cast-compatible, so the value types are pinned to the + // source columns: TraceId is String, AiSessionKeyHash is UInt64. + expect(sql).toContain("TracesTotal AggregateFunction(uniqCombined(12), String)") + expect(sql).toContain("TracesWithKey AggregateFunction(uniqCombined(12), String)") + expect(sql).toContain("SessionsApprox AggregateFunction(uniqCombined(12), UInt64)") + + expect(sql).toContain( + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_ai_vendors_hourly_mv TO service_ai_vendors_hourly", + ) + // The filter is the cost model and the semantics: non-AI spans never reach + // the HLL states, and post-enablement "no rows" means "no AI spans". + expect(sql).toContain("WHERE AiVendor != ''") + // The stored clamped hour, not toStartOfHour(Timestamp) — a skewed client + // must not be able to open a partition in 2038. + expect(sql).toContain("AiRollupHour AS Hour") + // Adjusted-count convention with the zero/unset floor. + expect(sql).toContain("sum(if(SampleRate > 0, SampleRate, 1.0)) AS WeightedSpanCount") + + // Correctness here depends on the source rows having been classified, not + // on the view having existed, so there is nothing safe to backfill and the + // §8 runbook gate (flag at 100% for a full clock hour) covers the boundary. + expect(sql).not.toContain("POPULATE") + expect(migration_0016_service_ai_vendors_hourly.statements.filter(isBackfill)).toHaveLength(0) + }) + + it("keeps the migrated AI rollup view identical to the deployed one", () => { + // A BYO cluster migrated to 16 and a freshly bootstrapped one must compute + // the same coverage ratio. Both bodies come from one exported constant; + // this asserts neither copy drifted away from it. + const migrationMv = migration_0016_service_ai_vendors_hourly.statements.find((statement) => + statement.includes("CREATE MATERIALIZED VIEW"), + ) + const snapshotMv = latestSnapshotStatements.find((statement) => + statement.includes("CREATE MATERIALIZED VIEW IF NOT EXISTS service_ai_vendors_hourly_mv"), + ) + + expect(migrationMv).toContain(SERVICE_AI_VENDORS_HOURLY_SELECT_SQL) + expect(snapshotMv).toContain(SERVICE_AI_VENDORS_HOURLY_SELECT_SQL) + }) + + it("keeps the AI rollup MV projection in the target's column order", () => { + // An MV writes into its target positionally. Every counter here is one of + // two ClickHouse types, so a reordered SELECT transposes columns silently — + // KeyAbsentCount and KeyInvalidCount would simply swap meanings. + const expectedOrder = [ + "OrgId", + "ServiceName", + "AiVendor", + "Hour", + "SpanCount", + "WeightedSpanCount", + "EligibleSpanCount", + "KeyAbsentCount", + "KeyInvalidCount", + "KeySubSessionCount", + "KeySessionCount", + "TracesTotal", + "TracesWithKey", + "SessionsApprox", + "RowRulesVersionMin", + "RowRulesVersionMax", + "RollupRulesVersion", + ] + + const createTable = migration_0016_service_ai_vendors_hourly.statements.find((statement) => + statement.startsWith("CREATE TABLE"), + )! + const tableColumns = createTable + .split("\n") + .slice(1, expectedOrder.length + 1) + .map((line) => line.trim().split(" ")[0]!) + + const selectColumns = SERVICE_AI_VENDORS_HOURLY_SELECT_SQL.split("\n") + .slice(1, expectedOrder.length + 1) + .map((line) => line.trim().replace(/,$/, "")) + .map((line) => (line.includes(" AS ") ? line.slice(line.lastIndexOf(" AS ") + 4) : line)) + + expect(tableColumns).toEqual(expectedOrder) + expect(selectColumns).toEqual(expectedOrder) + }) + it("adds the AI classification columns as defaulted trailing columns with no mutation", () => { const sql = migration_0015_ai_classification_columns.statements.join("\n") diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index d6742a6f3..1e55c16b4 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -14,6 +14,7 @@ import { migration_0012_session_event_attribute_keys } from "./0012_session_even import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events } from "./0014_web_events" import { migration_0015_ai_classification_columns } from "./0015_ai_classification_columns" +import { migration_0016_service_ai_vendors_hourly } from "./0016_service_ai_vendors_hourly" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -60,6 +61,7 @@ export const migrations: ReadonlyArray = [ migration_0013_service_map_ingest_bridge, migration_0014_web_events, migration_0015_ai_classification_columns, + migration_0016_service_ai_vendors_hourly, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 92a9bac8b..dba0d8f15 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c" as const +export const projectRevision = "097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -19,6 +19,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS metrics_histogram (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Count UInt64,\n Sum Float64,\n BucketCounts Array(UInt64),\n ExplicitBounds Array(Float64),\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n Flags UInt32,\n Min Nullable(Float64),\n Max Nullable(Float64),\n AggregationTemporality Int32\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_sum (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n AggregationTemporality Int32,\n IsMonotonic Bool\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService)\nTTL toDate(Hour) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS service_ai_vendors_hourly (\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n AiVendor LowCardinality(String),\n Hour DateTime('UTC'),\n SpanCount SimpleAggregateFunction(sum, UInt64),\n WeightedSpanCount SimpleAggregateFunction(sum, Float64),\n EligibleSpanCount SimpleAggregateFunction(sum, UInt64),\n KeyAbsentCount SimpleAggregateFunction(sum, UInt64),\n KeyInvalidCount SimpleAggregateFunction(sum, UInt64),\n KeySubSessionCount SimpleAggregateFunction(sum, UInt64),\n KeySessionCount SimpleAggregateFunction(sum, UInt64),\n TracesTotal AggregateFunction(uniqCombined(12), String),\n TracesWithKey AggregateFunction(uniqCombined(12), String),\n SessionsApprox AggregateFunction(uniqCombined(12), UInt64),\n RowRulesVersionMin SimpleAggregateFunction(min, UInt32),\n RowRulesVersionMax SimpleAggregateFunction(max, UInt32),\n RollupRulesVersion SimpleAggregateFunction(max, UInt32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMMDD(Hour)\nORDER BY (OrgId, ServiceName, AiVendor, Hour)\nTTL Hour + INTERVAL 400 DAY", "CREATE TABLE IF NOT EXISTS service_external_edges_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_map_children (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, ParentSpanId, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64),\n DbNamespace LowCardinality(String)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace)\nTTL toDate(Hour) + INTERVAL 365 DAY", @@ -54,6 +55,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_ai_vendors_hourly_mv TO service_ai_vendors_hourly AS\nSELECT\n OrgId,\n ServiceName,\n AiVendor,\n AiRollupHour AS Hour,\n count() AS SpanCount,\n sum(if(SampleRate > 0, SampleRate, 1.0)) AS WeightedSpanCount,\n countIf(AiSessionKeyState >= 3) AS EligibleSpanCount,\n countIf(AiSessionKeyState = 3) AS KeyAbsentCount,\n countIf(AiSessionKeyState = 4) AS KeyInvalidCount,\n countIf(AiSessionKeyState = 5) AS KeySubSessionCount,\n countIf(AiSessionKeyState = 6) AS KeySessionCount,\n uniqCombinedState(12)(TraceId) AS TracesTotal,\n uniqCombinedStateIf(12)(TraceId, AiSessionKeyState = 6) AS TracesWithKey,\n uniqCombinedStateIf(12)(AiSessionKeyHash, AiSessionKeyState = 6) AS SessionsApprox,\n min(AiRulesVersion) AS RowRulesVersionMin,\n max(AiRulesVersion) AS RowRulesVersionMax,\n max(AiRulesVersion) AS RollupRulesVersion\nFROM traces\nWHERE AiVendor != ''\nGROUP BY OrgId, ServiceName, AiVendor, Hour", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 418381e44..db0277f92 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "062342f168e1358e26e119c51cf59cd8628b250d8bc5152dc0d26927cf25c00c" as const +export const projectRevision = "097d8372a0ca33858cbd526a20931d50432f5a5a858544787df5db35daa770a7" as const export const datasources = [ { @@ -79,6 +79,11 @@ export const datasources = [ content: 'DESCRIPTION >\n Resolved (sourceService, parent.server.address) → resolved targetService facts emitted by the ServiceMapRollupService rollup. Used to anti-join internal-service overlap out of the external-edges query.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Hour DateTime `json:$.Hour`,\n SourceService LowCardinality(String) `json:$.SourceService`,\n ParentServerAddress String `json:$.ParentServerAddress`,\n ResolvedTargetService LowCardinality(String) `json:$.ResolvedTargetService`,\n DeploymentEnv LowCardinality(String) `json:$.DeploymentEnv`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService"\nENGINE_TTL "toDate(Hour) + INTERVAL 365 DAY"', }, + { + name: "service_ai_vendors_hourly", + content: + 'DESCRIPTION >\n Hourly per-service AI vendor discovery rollup: span/eligibility/session-key-state counters plus uniqCombined(12) trace and session states. AggregatingMergeTree MV target for AI service pruning, coverage and sampling-exemption suggestions.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n AiVendor LowCardinality(String),\n Hour DateTime(\'UTC\'),\n SpanCount SimpleAggregateFunction(sum, UInt64),\n WeightedSpanCount SimpleAggregateFunction(sum, Float64),\n EligibleSpanCount SimpleAggregateFunction(sum, UInt64),\n KeyAbsentCount SimpleAggregateFunction(sum, UInt64),\n KeyInvalidCount SimpleAggregateFunction(sum, UInt64),\n KeySubSessionCount SimpleAggregateFunction(sum, UInt64),\n KeySessionCount SimpleAggregateFunction(sum, UInt64),\n TracesTotal AggregateFunction(uniqCombined(12), String),\n TracesWithKey AggregateFunction(uniqCombined(12), String),\n SessionsApprox AggregateFunction(uniqCombined(12), UInt64),\n RowRulesVersionMin SimpleAggregateFunction(min, UInt32),\n RowRulesVersionMax SimpleAggregateFunction(max, UInt32),\n RollupRulesVersion SimpleAggregateFunction(max, UInt32)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMMDD(Hour)"\nENGINE_SORTING_KEY "OrgId, ServiceName, AiVendor, Hour"\nENGINE_TTL "Hour + INTERVAL 400 DAY"', + }, { name: "service_external_edges_hourly", content: @@ -257,6 +262,11 @@ export const pipes = [ content: "DESCRIPTION >\n Hourly rollup of distinct sum metrics into metric_catalog.\n\nNODE metric_catalog_sum_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", }, + { + name: "service_ai_vendors_hourly_mv", + content: + "DESCRIPTION >\n Aggregates classified AI spans hourly per service and vendor: span/eligibility/session-key-state counters plus uniqCombined(12) trace-coverage and session states. Filters AiVendor != '' so non-AI traffic never enters MV processing.\n\nNODE service_ai_vendors_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n AiVendor,\n AiRollupHour AS Hour,\n count() AS SpanCount,\n sum(if(SampleRate > 0, SampleRate, 1.0)) AS WeightedSpanCount,\n countIf(AiSessionKeyState >= 3) AS EligibleSpanCount,\n countIf(AiSessionKeyState = 3) AS KeyAbsentCount,\n countIf(AiSessionKeyState = 4) AS KeyInvalidCount,\n countIf(AiSessionKeyState = 5) AS KeySubSessionCount,\n countIf(AiSessionKeyState = 6) AS KeySessionCount,\n uniqCombinedState(12)(TraceId) AS TracesTotal,\n uniqCombinedStateIf(12)(TraceId, AiSessionKeyState = 6) AS TracesWithKey,\n uniqCombinedStateIf(12)(AiSessionKeyHash, AiSessionKeyState = 6) AS SessionsApprox,\n min(AiRulesVersion) AS RowRulesVersionMin,\n max(AiRulesVersion) AS RowRulesVersionMax,\n max(AiRulesVersion) AS RollupRulesVersion\n FROM traces\n WHERE AiVendor != ''\n GROUP BY OrgId, ServiceName, AiVendor, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_ai_vendors_hourly", + }, { name: "service_external_edges_hourly_mv", content: diff --git a/packages/domain/src/tinybird/ai-vendors-rollup-sql.ts b/packages/domain/src/tinybird/ai-vendors-rollup-sql.ts new file mode 100644 index 000000000..a40bc830a --- /dev/null +++ b/packages/domain/src/tinybird/ai-vendors-rollup-sql.ts @@ -0,0 +1,34 @@ +/** + * The single definition of the `service_ai_vendors_hourly` rollup SELECT. + * + * Shared by the materialized view (`serviceAiVendorsHourlyMv`) and ClickHouse + * migration 0016, which creates the same view on an already-deployed cluster. + * Two copies of this SELECT would be two chances for the deployed MV and the + * migrated one to disagree about what a coverage ratio means; the migration + * index test asserts they stay byte-identical. + * + * Column order matches `serviceAiVendorsHourly`'s schema order exactly — an MV + * writes into its target positionally, so a reordered SELECT silently transposes + * columns of compatible type (every counter here is one of two types). + */ +export const SERVICE_AI_VENDORS_HOURLY_SELECT_SQL = `SELECT + OrgId, + ServiceName, + AiVendor, + AiRollupHour AS Hour, + count() AS SpanCount, + sum(if(SampleRate > 0, SampleRate, 1.0)) AS WeightedSpanCount, + countIf(AiSessionKeyState >= 3) AS EligibleSpanCount, + countIf(AiSessionKeyState = 3) AS KeyAbsentCount, + countIf(AiSessionKeyState = 4) AS KeyInvalidCount, + countIf(AiSessionKeyState = 5) AS KeySubSessionCount, + countIf(AiSessionKeyState = 6) AS KeySessionCount, + uniqCombinedState(12)(TraceId) AS TracesTotal, + uniqCombinedStateIf(12)(TraceId, AiSessionKeyState = 6) AS TracesWithKey, + uniqCombinedStateIf(12)(AiSessionKeyHash, AiSessionKeyState = 6) AS SessionsApprox, + min(AiRulesVersion) AS RowRulesVersionMin, + max(AiRulesVersion) AS RowRulesVersionMax, + max(AiRulesVersion) AS RollupRulesVersion +FROM traces +WHERE AiVendor != '' +GROUP BY OrgId, ServiceName, AiVendor, Hour` diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index f23c41c97..f44852e15 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2141,3 +2141,122 @@ export const webEvents = defineDatasource("web_events", { }) export type WebEventsRow = InferRow + +/** + * AI discovery rollup: one row per (org, service, vendor, hour). + * + * Purposes: read-path service pruning, vendor-per-service lookup, session-key + * health / education surface, and sampling-exemption suggestions. Its MV filters + * `AiVendor != ''`, so the HLL states and aggregate expressions never run on the + * platform's plain HTTP/DB traffic — non-AI spans never enter MV processing. + * + * The MV runs synchronously inside the INSERT pipeline: this target's part-count + * and merge-lag alerts are **trace-ingestion health**, not side-table health. A + * rollup target that hits `too many parts` fails the INSERT into `traces`. + * + * ## Reader contract + * + * Plain `sum()` / `min()` / `max()` with `GROUP BY` for the + * SimpleAggregateFunction columns, `uniqCombinedMerge(12)` for the + * AggregateFunction states. Parts are never guaranteed merged — **no reader may + * assume one row per key, and no reader may use `FINAL`**. + * + * Headline coverage is trace-level AND service-level: + * `uniqCombinedMerge(12)(TracesWithKey) / uniqCombinedMerge(12)(TracesTotal)` + * grouped by `(OrgId, ServiceName)`, **merging across vendor rows**. Per-vendor + * ratios are diagnostic only: a trace mixing langchain + litellm counts in both + * vendors' `TracesTotal` but only langchain's `TracesWithKey`, so a vendor-level + * ratio systematically understates co-occurring passthrough vendors. Span-level + * state counters stay for diagnostics; span-level coverage is uncorrelated with + * customer reality (openrouter: 26/1773 spans carry the key on a fully + * resolvable trace). + * + * AI share of a service = this table's `SpanCount` over the service's total span + * count from `service_usage` / `traces_aggregates_hourly` — the denominator + * already exists in the warehouse, which is why no all-traffic watermark table + * ships alongside this one. + * + * ## Boundaries + * + * Hours before the recorded enablement hour (see + * `AI_VENDORS_ROLLUP_ENABLEMENT_HOUR_ENV` in `@maple/domain`'s ai-registry) do + * not exist for readers: the MV's creation hour is truncated while looking + * perfectly healthy. + * + * `Timestamp` is span *start* time, so for hour-straddling traces the + * key-bearing entry span sits in the earlier hour and the later hour is + * denominator-only — late hours are systematically depressed, not + * double-counted. Rare (traces are seconds long) and documented, not corrected. + * + * SOURCE TTL: 30d (`traces`). This table keeps 400 days — a deliberate, + * documented retention asymmetry: the rollup outlives the raw spans it was + * derived from and therefore can never be rebuilt past the raw horizon. + * Org deletion is `ALTER TABLE … DELETE WHERE OrgId = …` (well-pruned, OrgId is + * the sort-key prefix) — the one sanctioned mutation on this table. + * + * Populated by materialized view, not direct ingestion. + */ +export const serviceAiVendorsHourly = defineDatasource("service_ai_vendors_hourly", { + description: + "Hourly per-service AI vendor discovery rollup: span/eligibility/session-key-state counters plus uniqCombined(12) trace and session states. AggregatingMergeTree MV target for AI service pruning, coverage and sampling-exemption suggestions.", + jsonPaths: false, + schema: { + /** Must match `traces.OrgId` exactly — an MV column type mismatch is a silent cast. */ + OrgId: t.string().lowCardinality(), + ServiceName: t.string().lowCardinality(), + /** Normalized vendor slug from the closed allowlist. Never '' — the MV filters those out. */ + AiVendor: t.string().lowCardinality(), + /** `traces.AiRollupHour`: receive-time-clamped and written by ingest, so deterministic. */ + Hour: t.dateTime("UTC"), + SpanCount: t.simpleAggregateFunction("sum", t.uint64()), + /** + * Sample-corrected span count. Maple's `SampleRate` is the **adjusted-count** + * convention (the schema notes say "multiply counts by SampleRate for + * unbiased throughput estimates"), so `sum(SampleRate)` is the unbiased + * estimate — NOT `sum(1/SampleRate)`, which is the convention for + * probability-semantics columns. The two invert each other; pinned here. + * The MV floors unset/zero rows to 1.0. + */ + WeightedSpanCount: t.simpleAggregateFunction("sum", t.float64()), + /** States 3..6 — spans where a session key was actually expected. */ + EligibleSpanCount: t.simpleAggregateFunction("sum", t.uint64()), + /** State 3 — eligible, no key present. */ + KeyAbsentCount: t.simpleAggregateFunction("sum", t.uint64()), + /** State 4 — a key was present but did not validate. */ + KeyInvalidCount: t.simpleAggregateFunction("sum", t.uint64()), + /** State 5 — a valid key at sub-session granularity (thread/run, not session). */ + KeySubSessionCount: t.simpleAggregateFunction("sum", t.uint64()), + /** State 6 — a valid session-granularity key. The only state that feeds coverage. */ + KeySessionCount: t.simpleAggregateFunction("sum", t.uint64()), + /** + * uniqCombined(12) ≈ 1.7% error, which caps HLL state size; a health ratio + * does not need better. State types are not cast-compatible, so the value + * type is pinned to `traces.TraceId`'s exact type (String). + */ + TracesTotal: t.aggregateFunction("uniqCombined(12)", t.string()), + /** Distinct traces with at least one state-6 span, same value type as TracesTotal. */ + TracesWithKey: t.aggregateFunction("uniqCombined(12)", t.string()), + /** Distinct `AiSessionKeyHash` values over state-6 spans (`cityHash64` of the value). */ + SessionsApprox: t.aggregateFunction("uniqCombined(12)", t.uint64()), + /** Provenance of the *rows*: the classifier versions that wrote them. */ + RowRulesVersionMin: t.simpleAggregateFunction("min", t.uint32()), + RowRulesVersionMax: t.simpleAggregateFunction("max", t.uint32()), + /** + * Provenance of the *counts*: the version of the writer that produced this + * aggregate. For MV-written rows it equals `RowRulesVersionMax` by + * construction; divergence means the partition was rebuilt by a later + * registry version over older rows, which is exactly the fact a rebuild + * needs to be able to state instead of hide. + */ + RollupRulesVersion: t.simpleAggregateFunction("max", t.uint32()), + }, + engine: engine.aggregatingMergeTree({ + // Daily partitions: rebuild after a registry fix is a per-closed-day + // `ALTER TABLE … REPLACE PARTITION … FROM `, which is atomic. + partitionKey: "toYYYYMMDD(Hour)", + sortingKey: ["OrgId", "ServiceName", "AiVendor", "Hour"], + ttl: "Hour + INTERVAL 400 DAY", + }), +}) + +export type ServiceAiVendorsHourlyRow = InferRow diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index d0cfe296e..90add036f 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -25,6 +25,7 @@ import { serviceOperationsMinutely, serviceOperationsHourly, webEvents, + serviceAiVendorsHourly, } from "./datasources" import { DB_NAMESPACE_ATTR_SQL, @@ -34,6 +35,7 @@ import { DB_SYSTEM_ATTR_SQL, } from "./db-query-shape-sql" import { NORMALIZED_SPAN_NAME_SQL } from "./span-display-name" +import { SERVICE_AI_VENDORS_HOURLY_SELECT_SQL } from "./ai-vendors-rollup-sql" /** * Materialized view to aggregate log usage statistics per service per hour @@ -1423,3 +1425,40 @@ export const webEventsMv = defineMaterializedView("web_events_mv", { }), ], }) + +/** + * Populates `service_ai_vendors_hourly` — one row per (org, service, vendor, + * hour) — from classified AI spans. + * + * `WHERE AiVendor != ''` is the whole cost story: the platform's HTTP/DB traffic + * never reaches the aggregate expressions or the HLL states, so this MV's cost + * scales with AI traffic, not total traffic. It is also what makes "no rows for a + * service-hour" mean *genuinely no AI spans* post-enablement, since production + * classifies unconditionally. + * + * `Hour` is `traces.AiRollupHour` — a receive-time-clamped hour that ingest + * writes for every span, flag on or off. Grouping on a stored column rather than + * `toStartOfHour(Timestamp)` is what keeps a clock-skewed client from opening + * partitions in 2038. + * + * `WeightedSpanCount` sums `SampleRate` because Maple's `SampleRate` is the + * adjusted-count convention (see the column docs on the target); the + * `if(> 0, …, 1.0)` floor guards unset/zero rows so the sum stays finite. + * + * No `POPULATE` — the emitter never emits one, and the deploy runbook (§8 of the + * write-side plan) requires the classifier at 100% for a full clock hour before + * this view exists, so there is no window to backfill. + */ +export const serviceAiVendorsHourlyMv = defineMaterializedView("service_ai_vendors_hourly_mv", { + description: + "Aggregates classified AI spans hourly per service and vendor: span/eligibility/session-key-state counters plus uniqCombined(12) trace-coverage and session states. Filters AiVendor != '' so non-AI traffic never enters MV processing.", + datasource: serviceAiVendorsHourly, + nodes: [ + node({ + name: "service_ai_vendors_hourly_mv_node", + sql: ` + ${SERVICE_AI_VENDORS_HOURLY_SELECT_SQL} + `, + }), + ], +}) diff --git a/packages/domain/src/tinybird/retention-matrix.test.ts b/packages/domain/src/tinybird/retention-matrix.test.ts index 490c4c036..2dd09fcc6 100644 --- a/packages/domain/src/tinybird/retention-matrix.test.ts +++ b/packages/domain/src/tinybird/retention-matrix.test.ts @@ -17,6 +17,11 @@ const RETENTION_DAYS = { metrics_histogram: 90, metrics_sum: 90, service_address_resolutions_hourly: 365, + // The one 400-day tier, and deliberately so: a year-over-year vendor-adoption + // view needs the year *plus* slack, and the row count is a few thousand per + // org. It also outlives its 30-day source, which is the constraint that comes + // with the privilege — past the raw horizon these rows cannot be rebuilt. + service_ai_vendors_hourly: 400, service_external_edges_hourly: 365, service_map_children: 30, service_map_db_edges_hourly: 365, @@ -45,7 +50,7 @@ const RETENTION_DAYS = { const ZERO_RETENTION_DATASOURCES = ["service_map_edges_hourly_ingest"] as const describe("Tinybird retention matrix", () => { - it("assigns every stored datasource to the intended 30/90/365-day tier", () => { + it("assigns every stored datasource to the intended 30/90/365/400-day tier", () => { const actualNames = tinybirdProjectManifest.datasources.map(({ name }) => name).sort() expect(actualNames).toEqual([...Object.keys(RETENTION_DAYS), ...ZERO_RETENTION_DATASOURCES].sort()) diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index 328c60c6a..644acbc02 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -19,6 +19,9 @@ import { LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST_DIGEST, LOCAL_SCHEMA_V5_SQL, + LOCAL_SCHEMA_V6, + LOCAL_SCHEMA_V6_MANIFEST_DIGEST, + LOCAL_SCHEMA_V6_SQL, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -119,6 +122,18 @@ if ( fail("the immutable local schema v5 snapshot no longer matches its historical identity") } +const v6 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V6.version) +if ( + !v6 || + LOCAL_SCHEMA_V6_MANIFEST_DIGEST !== v6.manifestDigest || + schemaFingerprint(LOCAL_SCHEMA_V6_SQL) !== v6.fingerprint || + schemaDigest(LOCAL_SCHEMA_V6_SQL) !== v6.digest || + LOCAL_SCHEMA_V6.fingerprint !== v6.fingerprint || + LOCAL_SCHEMA_V6.digest !== v6.digest +) { + fail("the immutable local schema v6 snapshot no longer matches its historical identity") +} + const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name) if (new Set(names).size !== names.length) fail("local structural schema manifest contains duplicate object names")