From e6218443acf7b18ec54e00d6df2f37e378243a81 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Tue, 11 Aug 2026 22:18:20 +0200 Subject: [PATCH] feat(domain): add the typed registry view and its ClickHouse SQL compiler Adds `@maple/domain/ai-registry`: an Effect Schema decode of `registry.json` at module load (so a bad resync fails the import, not the first classification), the semantic invariants Schema cannot express (`validate.ts`: globally unique priorities, the D4 priority bands, the closed slug set, `value_prefix` confined to pseudo-keys), and `compile-sql.ts`, which lowers the registry to ClickHouse expressions over a `traces` row. The SQL compiler exists so the Rust classifier can be proven equivalent to a SQL evaluation of the same artifact, and so the rollup rebuild job has a way to recompute vendors without reading the `AiVendor` column it is fixing. It emits no hash SQL, but only because nothing in v1 consumes one: `sessionKeyValueExpr` yields the winning candidate's raw value and `cityHash64` over it is a one-liner the caller can write. The subpath export keeps the ~280 KB artifact off the root `@maple/domain` barrel that web and cli import, and `resolveJsonModule` is enabled for the vendored JSON. Co-Authored-By: Claude Fable 5 --- packages/domain/package.json | 1 + packages/domain/src/ai-registry/README.md | 27 ++ .../compile-sql.clickhouse.e2e.test.ts | 241 ++++++++++ .../src/ai-registry/compile-sql.test.ts | 315 +++++++++++++ .../domain/src/ai-registry/compile-sql.ts | 419 ++++++++++++++++++ packages/domain/src/ai-registry/index.ts | 55 +++ .../src/ai-registry/registry-fixtures.ts | 81 ++++ packages/domain/src/ai-registry/schema.ts | 278 ++++++++++++ .../domain/src/ai-registry/validate.test.ts | 294 ++++++++++++ packages/domain/src/ai-registry/validate.ts | 304 +++++++++++++ packages/domain/tsconfig.json | 2 + 11 files changed, 2017 insertions(+) create mode 100644 packages/domain/src/ai-registry/compile-sql.clickhouse.e2e.test.ts create mode 100644 packages/domain/src/ai-registry/compile-sql.test.ts create mode 100644 packages/domain/src/ai-registry/compile-sql.ts create mode 100644 packages/domain/src/ai-registry/index.ts create mode 100644 packages/domain/src/ai-registry/registry-fixtures.ts create mode 100644 packages/domain/src/ai-registry/schema.ts create mode 100644 packages/domain/src/ai-registry/validate.test.ts create mode 100644 packages/domain/src/ai-registry/validate.ts diff --git a/packages/domain/package.json b/packages/domain/package.json index bc0a373e1..20fd5a23c 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -5,6 +5,7 @@ "type": "module", "exports": { ".": "./src/index.ts", + "./ai-registry": "./src/ai-registry/index.ts", "./anticipated-errors": "./src/anticipated-errors.ts", "./billing": "./src/billing.ts", "./glob": "./src/glob.ts", diff --git a/packages/domain/src/ai-registry/README.md b/packages/domain/src/ai-registry/README.md index e7e59e1ee..e433097e3 100644 --- a/packages/domain/src/ai-registry/README.md +++ b/packages/domain/src/ai-registry/README.md @@ -106,3 +106,30 @@ Consequently: Rust evaluators must agree span-for-span, and the algebra's invariants (unique priorities, band ordering, `value_prefix` pseudo-key restriction, session-state reduction) are asserted directly against `registry.json`. + +## TypeScript consumers + +The files next to this README are Maple's typed view of the artifact; they never modify it. + +- **`schema.ts`** — Effect Schema for the whole document, plus the decode. `registry.json` is decoded + at module load and exported deep-frozen as `aiRegistry`, so a re-sync that changes the shape fails + the import rather than the first classification. Also exports `AI_SESSION_KEY_STATE`, the frozen + 0–6 ladder. +- **`validate.ts`** — the semantic invariants Schema cannot express: globally unique integer + priorities, the D4 bands, the closed slug set with `unknown:` reserved, `value_prefix` confined to + pseudo-keys, justification on sufficient resource matchers. Run as assertions over the real + artifact in `validate.test.ts` — **that test is the gate a future re-sync must pass**. It reports + violations; it never repairs them. A violation means the upstream compiler in trace-capture + produced a bad artifact and must be fixed there. +- **`compile-sql.ts`** — compiles the registry to ClickHouse expressions over a `traces` row + (`compileAiRegistrySql`, `renderAiRegistrySelect`). Two consumers: the Rust/SQL differential + tests, and the rollup rebuild job, which cannot read the stored `AiVendor` because that is the + column being fixed. No hash SQL is emitted: `sessionKeyValueExpr` yields the raw value and the + caller wraps it in `cityHash64` if it wants the hash. +- **`compile-sql.clickhouse.e2e.test.ts`** — runs the generated SQL through a real ClickHouse + (`bun ch:up`, then `CLICKHOUSE_E2E=1`). Parse/type-check/execute only; per-span semantics belong + to the differential suite. + +Imported as `@maple/domain/ai-registry`. Deliberately not re-exported from the root `@maple/domain` +barrel: everything here is pure data and string building and is safe for web/cli, but nobody should +pay ~280 KB of vendored JSON implicitly. diff --git a/packages/domain/src/ai-registry/compile-sql.clickhouse.e2e.test.ts b/packages/domain/src/ai-registry/compile-sql.clickhouse.e2e.test.ts new file mode 100644 index 000000000..ae5657523 --- /dev/null +++ b/packages/domain/src/ai-registry/compile-sql.clickhouse.e2e.test.ts @@ -0,0 +1,241 @@ +// The analyzer gate for the compiled registry. +// +// `./compile-sql.test.ts` asserts on SQL text, and text is not a contract ClickHouse honours — the +// repo has already shipped an expression that produced exactly the intended string and was then +// rejected with `NO_COMMON_TYPE`. This suite runs the generated expressions through a real server. +// +// Scope, deliberately: it proves the SQL **parses, type-checks and executes** and that each +// expression resolves to the column type the `traces` schema expects. It does NOT assert per-span +// classification semantics — Rust/SQL differential equivalence is a later stage that needs the +// capture corpus, which is not vendored (see README.md). +// +// Gated on `CLICKHOUSE_E2E=1` exactly like the apps/api suites, so a plain `bun run test` never +// reaches for a server that isn't running: +// +// bun ch:up +// CLICKHOUSE_E2E=1 bun run --cwd packages/domain test -- compile-sql.clickhouse.e2e + +import { afterAll, beforeAll, describe, expect, it } from "vitest" +import { compileAiRegistrySql, renderAiRegistrySelect } from "./compile-sql" + +const clickhouseE2eEnabled = process.env.CLICKHOUSE_E2E === "1" +const clickhouseUrl = process.env.CLICKHOUSE_E2E_URL ?? "http://127.0.0.1:8123" +const clickhouseUser = process.env.CLICKHOUSE_E2E_USER ?? "maple" +const clickhousePassword = process.env.CLICKHOUSE_E2E_PASSWORD ?? "maple" + +/** + * Managed Tinybird is ClickHouse 24.12 with `use_variant_as_common_type = 0`, where a type mismatch + * between `multiIf` branches is a hard error. Modern local/CI servers default the setting ON and + * quietly resolve the same expression to a `Variant`, which would let this suite pass on SQL that + * fails in production. Pinned for the same reason the apps/api harness pins it. + */ +const ANALYZER_STRICTNESS: Record = { use_variant_as_common_type: "0" } + +const exec = async ( + sql: string, + database = "default", + settings: Record = {}, +): Promise => { + const query = new URLSearchParams({ database, ...settings }) + const response = await fetch(`${clickhouseUrl.replace(/\/$/, "")}/?${query.toString()}`, { + method: "POST", + redirect: "manual", + headers: { + "Content-Type": "text/plain", + "X-ClickHouse-User": clickhouseUser, + "X-ClickHouse-Key": clickhousePassword, + "X-ClickHouse-Database": database, + }, + body: sql, + }) + const body = await response.text() + if (!response.ok) throw new Error(`ClickHouse ${response.status}: ${body.slice(0, 1200)}`) + return body +} + +/** `DESCRIBE (SELECT …)` type-checks the whole query without reading a row. */ +const describeQuery = async (sql: string): Promise> => { + const body = await exec(`DESCRIBE (\n${sql}\n) FORMAT TabSeparated`, database, ANALYZER_STRICTNESS) + return body + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => { + const [name, type] = line.split("\t") + return { name: name ?? "", type: type ?? "" } + }) +} + +const database = `maple_ai_registry_e2e_${Date.now()}_${Math.random().toString(36).slice(2, 8)}` +const table = "traces_min" + +/** + * The seven `traces` columns the compiler reads, at their production types. A minimal table rather + * than the full migration set: the compiler only ever touches these, and the column *types* — in + * particular `Map(LowCardinality(String), String)`, which is what makes `startsWith` over + * `mapKeys(...)` a real question — are what the analyzer needs to see. + */ +const createTable = ` +CREATE TABLE ${table} ( + SpanName LowCardinality(String), + ScopeName String, + ScopeVersion String, + ScopeSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeAttributes Map(LowCardinality(String), String), + SpanAttributes Map(LowCardinality(String), String) +) ENGINE = MergeTree ORDER BY tuple()` + +/** One span per shape the algebra can hit, so the SELECT executes over non-trivial input. */ +const rows = [ + // Sufficient scope match. + { + SpanName: "Agent.run", + ScopeName: "openinference.instrumentation.agno", + ScopeVersion: "1.0.1", + ScopeSchemaUrl: "", + ResourceAttributes: {}, + ScopeAttributes: {}, + SpanAttributes: { "openinference.span.kind": "AGENT", "session.id": "s-1" }, + }, + // Promotion case: insufficient scope + a same-vendor attr hit. + { + SpanName: "chat_client", + ScopeName: "org.springframework.boot", + ScopeVersion: "4.1.0", + ScopeSchemaUrl: "", + ResourceAttributes: {}, + ScopeAttributes: {}, + SpanAttributes: { "spring.ai.kind": "chat_client", "spring.ai.chat.client.conversation.id": "c-1" }, + }, + // Negative promotion case: the same insufficient scope with no AI attribute at all. + { + SpanName: "POST", + ScopeName: "org.springframework.boot", + ScopeVersion: "4.1.0", + ScopeSchemaUrl: "", + ResourceAttributes: {}, + ScopeAttributes: {}, + SpanAttributes: { "http.request.method": "POST" }, + }, + // Present-but-empty: the case `!= ''` would collapse into "absent". + { + SpanName: "llm", + ScopeName: "custom", + ScopeVersion: "", + ScopeSchemaUrl: "", + ResourceAttributes: {}, + ScopeAttributes: {}, + SpanAttributes: { "gen_ai.operation.name": "" }, + }, + // Non-AI. + { + SpanName: "GET /health", + ScopeName: "@opentelemetry/instrumentation-http", + ScopeVersion: "0.57.0", + ScopeSchemaUrl: "", + ResourceAttributes: { "service.name": "web" }, + ScopeAttributes: {}, + SpanAttributes: { "http.request.method": "GET" }, + }, +] + +const compiled = compileAiRegistrySql() + +describe.skipIf(!clickhouseE2eEnabled)("compiled AI registry SQL against ClickHouse", () => { + beforeAll(async () => { + await exec(`CREATE DATABASE ${database}`) + await exec(createTable, database) + await exec( + `INSERT INTO ${table} FORMAT JSONEachRow\n${rows.map((row) => JSON.stringify(row)).join("\n")}`, + database, + ) + }, 60_000) + + afterAll(async () => { + await exec(`DROP DATABASE IF EXISTS ${database}`) + }, 30_000) + + it("type-checks the vendor expression to a String", async () => { + const columns = await describeQuery( + `SELECT ${compiled.vendorExpr} AS ${compiled.vendorAlias} FROM ${table}`, + ) + expect(columns).toHaveLength(1) + expect(columns[0]?.type).toBe("String") + }) + + it("type-checks the session-state expression to a UInt8", async () => { + const columns = await describeQuery( + `SELECT\n${compiled.vendorExpr} AS ${compiled.vendorAlias},\n${compiled.sessionStateExpr} AS ${compiled.sessionStateAlias}\nFROM ${table}`, + ) + expect(columns.map((column) => column.name)).toEqual([ + compiled.vendorAlias, + compiled.sessionStateAlias, + ]) + // The ladder is 0..6 and the storage column is UInt8; a wider type here means a branch + // resolved to something other than a small literal. + expect(columns[1]?.type).toBe("UInt8") + }) + + it("type-checks the layered select under analyzer strictness", async () => { + const columns = await describeQuery(renderAiRegistrySelect(compiled, table)) + expect(columns.map((column) => `${column.name}:${column.type}`)).toEqual([ + `${compiled.vendorAlias}:String`, + `${compiled.sessionStateAlias}:UInt8`, + "AiSessionKeyValueComputed:String", + "AiRulesVersionComputed:UInt32", + ]) + }) + + it("stays inside the query-tree node limit that a flat select list blows", async () => { + // Regression guard for the alias-inlining blow-up: this is the query shape the rebuild job + // will run, and it must survive the analyzer with the real 21-vendor registry. + const sql = renderAiRegistrySelect(compiled, table, { passthrough: ["SpanName"] }) + const columns = await describeQuery(sql) + expect(columns[0]?.name).toBe("SpanName") + expect(columns).toHaveLength(5) + }) + + it("executes over rows and returns one result per span", async () => { + const body = await exec( + `${renderAiRegistrySelect(compiled, table, { passthrough: ["SpanName"] })}\nORDER BY SpanName\nFORMAT JSONEachRow`, + database, + ANALYZER_STRICTNESS, + ) + const results = body + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => JSON.parse(line) as Record) + + expect(results).toHaveLength(rows.length) + // Semantics are the differential suite's job; all this asserts is that every expression + // produced a value of the right shape for every row. + for (const result of results) { + expect(typeof result[compiled.vendorAlias]).toBe("string") + expect(typeof result[compiled.sessionStateAlias]).toBe("number") + expect(typeof result.AiSessionKeyValueComputed).toBe("string") + expect(result.AiRulesVersionComputed).toBe(compiled.rulesVersion) + } + }) + + it("keeps present-but-empty distinguishable from absent", async () => { + // The one semantic claim worth making here, because it is a property of the *generated SQL* + // rather than of any vendor rule: `mapContains` must see the empty-valued key. + const body = await exec( + `SELECT ${compiled.vendorExpr} AS v FROM ${table} WHERE SpanName = 'llm' FORMAT TabSeparated`, + database, + ANALYZER_STRICTNESS, + ) + expect(body.trim()).toBe("unknown:genai") + }) + + it("compiles against overridden aliases", async () => { + const shadow = compileAiRegistrySql(undefined, { vendorAlias: "V", sessionStateAlias: "S" }) + const columns = await describeQuery(renderAiRegistrySelect(shadow, table)) + expect(columns.map((column) => column.name)).toEqual([ + "V", + "S", + "AiSessionKeyValueComputed", + "AiRulesVersionComputed", + ]) + }) +}) diff --git a/packages/domain/src/ai-registry/compile-sql.test.ts b/packages/domain/src/ai-registry/compile-sql.test.ts new file mode 100644 index 000000000..c06809e93 --- /dev/null +++ b/packages/domain/src/ai-registry/compile-sql.test.ts @@ -0,0 +1,315 @@ +// Compiler unit tests. +// +// These assert on SQL *text*, which is not a contract ClickHouse honours — that is what +// `./compile-sql.clickhouse.e2e.test.ts` is for. What text assertions are good at is pinning the +// algebra decisions that are easy to regress and impossible to see in a passing analyzer run: that +// presence never degrades to `!= ''`, that an insufficient scope match is AND-gated by its +// vendor's attr matchers, and that the branch order is global priority order. + +import { describe, expect, it } from "vitest" +import { + DEFAULT_AI_REGISTRY_SQL_COLUMNS, + compileAiRegistrySql, + compilePredicateSql, + renderAiRegistrySelect, +} from "./compile-sql" +import { aiRegistry } from "./schema" +import { testMatcher, testRegistry, testVendor, valuePrefixVendor } from "./registry-fixtures" + +const compiled = compileAiRegistrySql() + +describe("predicate algebra", () => { + it("compiles present to mapContains, never to a non-empty test", () => { + expect(compilePredicateSql({ op: "present", key: "gen_ai.operation.name" }, "span")).toBe( + "mapContains(SpanAttributes, 'gen_ai.operation.name')", + ) + }) + + it("keeps mapContains on eq so eq(key, '') stays distinguishable from an absent key", () => { + expect(compilePredicateSql({ op: "eq", key: "gen_ai.system", value: "" }, "span")).toBe( + "(SpanAttributes['gen_ai.system'] = '' AND mapContains(SpanAttributes, 'gen_ai.system'))", + ) + }) + + it("compiles key_prefix over the map's keys", () => { + expect(compilePredicateSql({ op: "key_prefix", prefix: "spring.ai." }, "span")).toBe( + "arrayExists(k -> startsWith(k, 'spring.ai.'), mapKeys(SpanAttributes))", + ) + }) + + it("resolves value_prefix pseudo-keys to real columns", () => { + expect( + compilePredicateSql({ op: "value_prefix", key: "scope.name", prefix: "openinference." }, "scope"), + ).toBe("startsWith(ScopeName, 'openinference.')") + expect(compilePredicateSql({ op: "value_prefix", key: "span.name", prefix: "Chat." }, "span")).toBe( + "startsWith(SpanName, 'Chat.')", + ) + }) + + it("targets the map named by the matcher class", () => { + const predicate = { op: "eq", key: "telemetry.sdk.name", value: "@mastra/otel-exporter" } as const + expect(compilePredicateSql(predicate, "resource")).toContain("ResourceAttributes[") + expect(compilePredicateSql(predicate, "scope")).toContain("ScopeAttributes[") + expect(compilePredicateSql(predicate, "span")).toContain("SpanAttributes[") + }) + + it("resolves pseudo-keys to columns regardless of the matcher class", () => { + // effect_ai's ATTR matchers key on span.name; class-only targeting would never fire. + expect(compilePredicateSql({ op: "eq", key: "span.name", value: "Chat.export" }, "span")).toBe( + "SpanName = 'Chat.export'", + ) + }) + + it("escapes quotes and backslashes in literals", () => { + expect(compilePredicateSql({ op: "eq", key: "k", value: "it's\\here" }, "span")).toContain( + "'it\\'s\\\\here'", + ) + }) + + it("honours column-name overrides", () => { + const shadow = compileAiRegistrySql(aiRegistry, { + columns: { spanAttributes: "Attrs", scopeName: "Scope" }, + }) + expect(shadow.vendorExpr).toContain("mapKeys(Attrs)") + expect(shadow.vendorExpr).toContain("Scope = 'litellm'") + expect(shadow.vendorExpr).not.toContain("SpanAttributes") + }) +}) + +describe("vendor resolution", () => { + it("emits a standalone branch for a sufficient scope matcher", () => { + expect(compiled.vendorExpr).toContain("\tScopeName = 'openinference.instrumentation.agno', 'agno'") + }) + + it("AND-gates an insufficient scope matcher with its own vendor's attr matchers", () => { + // spring_ai is the canonical promotion case: `org.springframework.boot` is Spring's global + // Micrometer scope, so a plain HTTP POST under it must stay non-AI. + expect(compiled.vendorExpr).toContain( + "\t(ScopeName = 'org.springframework.boot' AND (arrayExists(k -> startsWith(k, 'spring.ai.'), mapKeys(SpanAttributes)) OR (SpanAttributes['gen_ai.system'] = 'spring_ai' AND mapContains(SpanAttributes, 'gen_ai.system')))), 'spring_ai'", + ) + }) + + it("never emits a bare insufficient scope/resource condition", () => { + for (const vendor of aiRegistry.vendors) + for (const matcher of vendor.matchers) { + if (matcher.sufficient || matcher.class === "attr") continue + const bare = compilePredicateSql( + matcher.predicate, + matcher.class === "resource" ? "resource" : "scope", + DEFAULT_AI_REGISTRY_SQL_COLUMNS, + ) + expect(compiled.vendorExpr).not.toContain(`\t${bare}, '${vendor.vendor}'`) + } + }) + + it("drops a conditional candidate no attr matcher could promote", () => { + const registry = testRegistry([ + testVendor({ + vendor: "orphan", + matchers: [ + testMatcher({ + class: "scope", + sufficient: false, + priority: 29_000, + predicate: { op: "eq", key: "scope.name", value: "generic" }, + }), + ], + }), + ]) + expect(compileAiRegistrySql(registry).vendorExpr).toBe("''") + }) + + it("emits attr matchers as standalone branches at their own priority", () => { + expect(compiled.vendorExpr).toContain( + "\tarrayExists(k -> startsWith(k, 'spring.ai.'), mapKeys(SpanAttributes)), 'spring_ai'", + ) + }) + + it("compiles value_prefix vendors", () => { + const sql = compileAiRegistrySql(testRegistry([valuePrefixVendor])).vendorExpr + expect(sql).toContain("startsWith(ScopeName, 'openinference.instrumentation.'), 'prefixed_vendor'") + }) + + it("orders every branch by descending global priority", () => { + const order = [ + ...aiRegistry.vendors.flatMap((vendor) => + vendor.matchers.map((matcher) => ({ priority: matcher.priority, slug: vendor.vendor })), + ), + ...aiRegistry.unknown_tier.map((rule) => ({ priority: rule.priority, slug: rule.bucket })), + ] + .sort((left, right) => right.priority - left.priority) + .map((entry) => entry.slug) + + const emitted = compiled.vendorExpr + .split("\n") + .map((line) => /, '([^']*)'$/.exec(line.trim().replace(/,$/, ""))) + .flatMap((match) => (match?.[1] === undefined ? [] : [match[1]])) + + // Every emitted slug appears in priority order; dropped branches (unpromotable candidates) + // are absent from the emitted list but never reorder what remains. + let cursor = -1 + for (const slug of emitted) { + const next = order.indexOf(slug, cursor + 1) + expect(next).toBeGreaterThan(cursor) + cursor = next + } + }) + + it("ranks the unknown tier below every vendor branch", () => { + const lines = compiled.vendorExpr.split("\n") + const firstUnknown = lines.findIndex((line) => line.includes("'unknown:")) + const lastVendor = lines.reduce( + (last, line, index) => + /, '(?!unknown:)[a-z]/.test(line) && !line.includes("'unknown:") ? index : last, + -1, + ) + expect(firstUnknown).toBeGreaterThan(lastVendor) + }) + + it("falls back to '' — the definitive non-AI answer", () => { + expect(compiled.vendorExpr.trimEnd().endsWith("\t''\n)")).toBe(true) + }) + + it("never uses != '' for presence anywhere in vendor resolution", () => { + expect(compiled.vendorExpr).not.toContain("!= ''") + }) + + it("never uses lowerUTF8/upperUTF8 (chdb lint ban)", () => { + const all = compiled.vendorExpr + compiled.sessionStateExpr + compiled.sessionKeyValueExpr + expect(all).not.toMatch(/lowerUTF8|upperUTF8/) + }) +}) + +describe("session state", () => { + it("compiles a single-candidate vendor to the full ladder", () => { + expect(compiled.sessionStateExpr).toContain( + [ + "\tAiVendorComputed = 'spring_ai', multiIf(", + "\t\tNOT ((SpanAttributes['spring.ai.kind'] = 'chat_client' AND mapContains(SpanAttributes, 'spring.ai.kind'))), 2,", + "\t\tNOT mapContains(SpanAttributes, 'spring.ai.chat.client.conversation.id'), 3,", + "\t\t(SpanAttributes['spring.ai.chat.client.conversation.id'] = '' OR SpanAttributes['spring.ai.chat.client.conversation.id'] IN ('default')), 4,", + "\t\t6", + "\t)", + ].join("\n"), + ) + }) + + it("reduces multiple candidates with greatest()", () => { + expect(compiled.sessionStateExpr).toContain("AiVendorComputed = 'agno', greatest(") + // agno: session.id at session granularity (6) and agno.run.id at run granularity (5). + expect(compiled.sessionStateExpr).toContain("mapContains(SpanAttributes, 'agno.run.id')") + }) + + it("omits the authority branch when every span is authoritative", () => { + const registry = testRegistry([ + testVendor({ + vendor: "a", + matchers: [testMatcher({ priority: 29_000, predicate: { op: "present", key: "a.x" } })], + session_candidates: [ + { + key: "session.id", + authority_predicate: null, + validation: ["non_empty"], + granularity: "session", + verdict: "A", + }, + ], + }), + ]) + const state = compileAiRegistrySql(registry).sessionStateExpr + expect(state).not.toContain(", 2,") + expect(state).toContain("NOT mapContains(SpanAttributes, 'session.id'), 3,") + }) + + it("resolves sub-session granularity to 5 and session to 6", () => { + const build = (granularity: "session" | "run") => + compileAiRegistrySql( + testRegistry([ + testVendor({ + vendor: "a", + matchers: [ + testMatcher({ priority: 29_000, predicate: { op: "present", key: "a.x" } }), + ], + session_candidates: [ + { + key: "k", + authority_predicate: null, + validation: [], + granularity, + verdict: "A", + }, + ], + }), + ]), + ).sessionStateExpr + expect(build("session")).toContain("\t\t6\n") + expect(build("run")).toContain("\t\t5\n") + }) + + it("falls back to 0 with no vendor and 1 for a vendor without session rules", () => { + expect(compiled.sessionStateExpr).toContain("if(AiVendorComputed = '', 0, 1)") + }) + + it("gives every unknown:* bucket state 1 by falling through", () => { + for (const rule of aiRegistry.unknown_tier) + expect(compiled.sessionStateExpr).not.toContain(`AiVendorComputed = '${rule.bucket}'`) + }) +}) + +describe("session key value", () => { + it("selects the first candidate whose state equals the reduced state", () => { + expect(compiled.sessionKeyValueExpr).toContain("if(AiSessionKeyStateComputed >= 5, multiIf(") + expect(compiled.sessionKeyValueExpr).toContain( + "= AiSessionKeyStateComputed, SpanAttributes['session.id']", + ) + }) + + it("yields the raw value, leaving the hash to the caller", () => { + expect(compiled.sessionKeyValueExpr).not.toMatch(/cityHash64/) + expect(compiled.vendorExpr).not.toMatch(/cityHash64/) + }) +}) + +describe("output shape", () => { + it("carries the registry version", () => { + expect(compiled.rulesVersion).toBe(aiRegistry.registry_version) + expect(compiled.rulesVersionExpr).toBe(`toUInt32(${aiRegistry.registry_version})`) + }) + + it("references the vendor and state aliases from the session expressions", () => { + expect(compiled.sessionStateExpr).toContain(compiled.vendorAlias) + expect(compiled.sessionKeyValueExpr).toContain(compiled.sessionStateAlias) + }) + + it("is deterministic: compiling twice yields byte-identical SQL", () => { + const again = compileAiRegistrySql() + expect(again.vendorExpr).toBe(compiled.vendorExpr) + expect(again.sessionStateExpr).toBe(compiled.sessionStateExpr) + expect(again.sessionKeyValueExpr).toBe(compiled.sessionKeyValueExpr) + expect(renderAiRegistrySelect(again, "traces")).toBe(renderAiRegistrySelect(compiled, "traces")) + }) + + it("layers the three expressions in nested subqueries, each built exactly once", () => { + // The regression guard for `Query tree is too big`: ClickHouse inlines a SELECT alias at + // every reference, so a flat select list expands sessionStateExpr once per candidate. A + // subquery boundary makes each alias a real output column instead. + const sql = renderAiRegistrySelect(compiled, "traces", { passthrough: ["OrgId", "TraceId"] }) + const occurrences = (needle: string): number => sql.split(needle).length - 1 + + expect(occurrences(`AS ${compiled.vendorAlias}`)).toBe(1) + expect(occurrences(`AS ${compiled.sessionStateAlias}`)).toBe(1) + expect(occurrences("AS AiSessionKeyValueComputed")).toBe(1) + // Three projections: outer, state, vendor. + expect(occurrences("SELECT")).toBe(3) + expect(sql).toContain("FROM traces") + expect(sql.indexOf("OrgId")).toBeLessThan(sql.indexOf(`AS ${compiled.sessionStateAlias}`)) + }) + + it("balances its parentheses", () => { + for (const sql of [compiled.vendorExpr, compiled.sessionStateExpr, compiled.sessionKeyValueExpr]) { + const opens = (sql.match(/\(/g) ?? []).length + const closes = (sql.match(/\)/g) ?? []).length + expect(opens).toBe(closes) + } + }) +}) diff --git a/packages/domain/src/ai-registry/compile-sql.ts b/packages/domain/src/ai-registry/compile-sql.ts new file mode 100644 index 000000000..42b08018a --- /dev/null +++ b/packages/domain/src/ai-registry/compile-sql.ts @@ -0,0 +1,419 @@ +// Compiles the AI-vendor registry to ClickHouse expressions over a `traces` row. +// +// Two consumers justify this existing at all: +// +// 1. **Equivalence testing.** The Rust detector in `apps/ingest` classifies at write time. These +// expressions re-derive the same answer from the row it wrote, so a differential test can +// compare the two over real captures (plan §6). SQL text that merely *looks* right is not the +// contract — the analyzer is, which is why `./compile-sql.clickhouse.e2e.test.ts` runs the +// output through a real ClickHouse. +// 2. **Rebuilds.** When a registry fix means "previously unclassified spans now classify", the +// rollup rebuild job (plan §4) recomputes `service_ai_vendors_hourly` from raw rows. It cannot +// use the stored `AiVendor` — that is the column being fixed — so it needs the rules as SQL. +// +// **No hash SQL is emitted — a scope boundary, not a constraint.** `AiSessionKeyHash` is +// `cityHash64(value)` over the winning candidate's raw value, which `sessionKeyValueExpr` already +// exposes, so wrapping it is a one-liner any caller can write and the equivalence suite's SQL leg +// does exactly that. It is not emitted here because nothing in v1 consumes it: the rollup rebuild +// job is deferred with the rest of the read path. Note that plan §4 declares `SessionsApprox` **not +// rebuildable** when a registry fix changes which candidate wins — that does not hold here. The +// rebuild can recompute the hash from the value this compiler already exposes, and +// `hash-alignment.clickhouse.e2e.test.ts` proves ClickHouse's `cityHash64` returns exactly what the +// ingest writer wrote, so recomputed hashes merge with MV-written ones. The real limit on a rebuild +// is the raw horizon: `traces` keeps 30 days and this rollup keeps 400. +// +// ## Predicate targeting +// +// The pseudo-keys `scope.name` / `span.name` / `scope.version` / `scope.schema_url` are real +// columns and always resolve to those columns, whatever class the matcher carries — `effect_ai`'s +// attr matchers key on `span.name`, so class-only targeting would silently never fire. Every other +// key resolves to one map, chosen by matcher class: `resource` → ResourceAttributes, +// `scope` → ScopeAttributes, `attr` → SpanAttributes. Predicates with no class (session-candidate +// authority predicates, candidate key lookups, unknown-tier fingerprints) are span-local and read +// SpanAttributes. +// +// This is the **shared** semantics: `apps/ingest/src/ai_classifier.rs` resolves matchers the same +// way, and the differential suites in this directory hold the two to it (0 pinned divergences). +// +// The outlier is trace-capture's `scripts/verify-seed.ts`, whose `lookup()` falls back +// span → scope → resource for every key regardless of class and unions all three attribute lists +// as `key_prefix` evidence. It keeps that fallback, in its own repo: it verifies one seed at a +// time, where a cross-class read cannot promote some *other* vendor. Here it could, and did — +// `langsmith.internal_provider` is langchain's insufficient resource key and also sits inside +// langchain's attr-class `key_prefix('langsmith.')`, so under the fallback the resource attribute +// alone satisfied the attr matcher, which promotes, and every span of that process — plain HTTP +// included, value ignored — classified `langchain`. Class-directed targeting is what makes plan +// §1's sufficiency gate ("an insufficient resource match contributes no hit on its own") hold. +// The capture corpus is insensitive to the difference: all 10,091 corpus spans classify +// identically under either rule, so no golden depends on the fallback. +// +// ## Presence +// +// `present` compiles to `mapContains`, never `!= ''`. Present-but-empty must stay distinguishable +// from absent — session-key state 4 ("key present but failed validation") exists precisely to +// separate them, and `!= ''` would collapse it into state 3. +// +// ## Why the expressions must be layered in subqueries +// +// The session expressions reference the computed vendor and state by SELECT alias. ClickHouse's +// analyzer **inlines** an alias at every reference rather than evaluating it once, so putting all +// three in one flat SELECT expands `sessionStateExpr` (which references the vendor alias 21 times) +// into `sessionKeyValueExpr` (which references the state alias once per candidate) and the query +// tree passes the 500k-node limit — `Query tree is too big`, measured on the real registry against +// ClickHouse 26.7, not theorised. `renderAiRegistrySelect` therefore emits three nested SELECTs: +// across a subquery boundary an alias is a real output column, so each expression is built once. +// Callers assembling the SQL themselves must preserve that layering. + +import { aiRegistry } from "./schema" +import type { AiRegistryDocument, Predicate, SessionCandidate, Vendor } from "./schema" +import { AI_SESSION_KEY_STATE } from "./schema" + +/** Column names on the row the expressions read. Overridable for shadow/staging tables. */ +export interface AiRegistrySqlColumns { + readonly spanAttributes: string + readonly scopeAttributes: string + readonly resourceAttributes: string + readonly spanName: string + readonly scopeName: string + readonly scopeVersion: string + readonly scopeSchemaUrl: string +} + +export const DEFAULT_AI_REGISTRY_SQL_COLUMNS: AiRegistrySqlColumns = { + spanAttributes: "SpanAttributes", + scopeAttributes: "ScopeAttributes", + resourceAttributes: "ResourceAttributes", + spanName: "SpanName", + scopeName: "ScopeName", + scopeVersion: "ScopeVersion", + scopeSchemaUrl: "ScopeSchemaUrl", +} + +export interface CompileAiRegistrySqlOptions { + readonly columns?: Partial + /** + * SELECT alias the session expressions reference for the computed vendor. It must be the + * *computed* vendor, not the stored `AiVendor` column — a rebuild exists because the stored + * value is wrong. + */ + readonly vendorAlias?: string + /** SELECT alias for the computed session state, referenced by `sessionKeyValueExpr`. */ + readonly sessionStateAlias?: string +} + +export interface CompiledAiRegistrySql { + /** Resolves to the vendor slug, an `unknown:*` bucket, or `''` for non-AI. */ + readonly vendorExpr: string + /** Resolves to the `AiSessionKeyState` ladder value. References `vendorAlias`. */ + readonly sessionStateExpr: string + /** + * The winning candidate's raw session-key value, or `''` below state 5. References both + * aliases. Never hashed here — see the module header. + */ + readonly sessionKeyValueExpr: string + /** `registry_version`, written to `traces.AiRulesVersion`. */ + readonly rulesVersion: number + /** The same value as a typed SQL literal. */ + readonly rulesVersionExpr: string + readonly columns: AiRegistrySqlColumns + readonly vendorAlias: string + readonly sessionStateAlias: string +} + +/** ClickHouse single-quoted string literal. Backslash first, or the quote escape is re-escaped. */ +export const quoteSqlString = (value: string): string => + `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` + +type AttributeTarget = "span" | "scope" | "resource" + +const targetColumn = (target: AttributeTarget, columns: AiRegistrySqlColumns): string => + target === "span" + ? columns.spanAttributes + : target === "scope" + ? columns.scopeAttributes + : columns.resourceAttributes + +const targetForClass = (matcherClass: string): AttributeTarget => { + if (matcherClass === "resource") return "resource" + if (matcherClass === "scope") return "scope" + if (matcherClass === "attr") return "span" + // `event` / `event_attr` are reserved in the schema and unimplemented in v1 (plan §1). Failing + // here is the point: a registry that starts using them must not compile to expressions that + // quietly ignore the rule. + throw new Error(`ai-registry: matcher class '${matcherClass}' is not implemented in v1`) +} + +const pseudoKeyColumn = (key: string, columns: AiRegistrySqlColumns): string | undefined => { + switch (key) { + case "scope.name": + return columns.scopeName + case "span.name": + return columns.spanName + case "scope.version": + return columns.scopeVersion + case "scope.schema_url": + return columns.scopeSchemaUrl + default: + return undefined + } +} + +/** `mapContains`, or `1` for a pseudo-key: a real column is always "present" (possibly empty). */ +const presenceExpr = (key: string, target: AttributeTarget, columns: AiRegistrySqlColumns): string => + pseudoKeyColumn(key, columns) !== undefined + ? "1" + : `mapContains(${targetColumn(target, columns)}, ${quoteSqlString(key)})` + +/** The canonical string value of `key`, as either a real column or a map lookup. */ +const valueExpr = (key: string, target: AttributeTarget, columns: AiRegistrySqlColumns): string => + pseudoKeyColumn(key, columns) ?? `${targetColumn(target, columns)}[${quoteSqlString(key)}]` + +export const compilePredicateSql = ( + predicate: Predicate, + target: AttributeTarget, + columns: AiRegistrySqlColumns = DEFAULT_AI_REGISTRY_SQL_COLUMNS, +): string => { + switch (predicate.op) { + case "present": + return presenceExpr(predicate.key, target, columns) + case "eq": { + const literal = quoteSqlString(predicate.value) + const pseudo = pseudoKeyColumn(predicate.key, columns) + if (pseudo !== undefined) return `${pseudo} = ${literal}` + // `mapContains` is redundant for a non-empty literal but load-bearing for `eq(key, '')`, + // where a missing key also reads as `''`. + return `(${valueExpr(predicate.key, target, columns)} = ${literal} AND ${presenceExpr(predicate.key, target, columns)})` + } + case "key_prefix": + return `arrayExists(k -> startsWith(k, ${quoteSqlString(predicate.prefix)}), mapKeys(${targetColumn(target, columns)}))` + case "value_prefix": + // Restricted to pseudo-keys by the algebra (D1), so this is always a real column. + return `startsWith(${valueExpr(predicate.key, target, columns)}, ${quoteSqlString(predicate.prefix)})` + } +} + +const anyOf = (conditions: ReadonlyArray): string => + conditions.length === 1 ? (conditions[0] as string) : `(${conditions.join(" OR ")})` + +const compileAuthoritySql = ( + candidate: SessionCandidate, + columns: AiRegistrySqlColumns, +): string | undefined => { + const authority = candidate.authority_predicate + if (authority === null) return undefined // every span of this vendor is authoritative + if ("any_of" in authority) + return anyOf(authority.any_of.map((predicate) => compilePredicateSql(predicate, "span", columns))) + return compilePredicateSql(authority, "span", columns) +} + +interface VendorBranch { + readonly priority: number + readonly vendor: string + readonly condition: string +} + +const collectVendorBranches = ( + registry: AiRegistryDocument, + columns: AiRegistrySqlColumns, +): ReadonlyArray => { + const branches: VendorBranch[] = [] + + for (const vendor of registry.vendors) { + const attrConditions = vendor.matchers + .filter((matcher) => matcher.class === "attr") + .map((matcher) => compilePredicateSql(matcher.predicate, "span", columns)) + + for (const matcher of vendor.matchers) { + const condition = compilePredicateSql(matcher.predicate, targetForClass(matcher.class), columns) + if (matcher.sufficient || matcher.class === "attr") { + // Unconditional hit at its own priority. + branches.push({ priority: matcher.priority, vendor: vendor.vendor, condition }) + continue + } + // The promotion rule: an insufficient resource/scope match is a conditional candidate, + // promoted only if the same span independently produces an attr hit for the SAME vendor. + // Without an attr matcher there is nothing that could ever promote it, so emitting the + // branch would classify Spring Boot's plain HTTP POSTs as spring_ai — the negative case + // plan §2 names explicitly. + if (attrConditions.length === 0) continue + branches.push({ + priority: matcher.priority, + vendor: vendor.vendor, + condition: `(${condition} AND ${anyOf(attrConditions)})`, + }) + } + } + + for (const rule of registry.unknown_tier) + branches.push({ + priority: rule.priority, + vendor: rule.bucket, + condition: compilePredicateSql(rule.predicate, "span", columns), + }) + + // Descending global priority — the same order the Rust resolver walks. Priorities are unique + // (asserted in `./validate`), so this sort is total and the output is deterministic. + return [...branches].sort((left, right) => right.priority - left.priority) +} + +/** One level of nesting, so a generated expression stays readable when a human has to debug it. */ +const indent = (sql: string): string => sql.split("\n").join("\n\t") + +const multiIf = (pairs: ReadonlyArray, fallback: string): string => { + if (pairs.length === 0) return fallback + const body = pairs.map(([condition, result]) => `\t${indent(condition)}, ${indent(result)}`).join(",\n") + return `multiIf(\n${body},\n\t${indent(fallback)}\n)` +} + +/** + * Per-candidate state, exactly the ladder in `AI_SESSION_KEY_STATE`: + * not authoritative → 2, key absent → 3, present but invalid → 4, else 6 at session granularity + * and 5 at run/user/instance. + */ +const compileCandidateStateSql = ( + candidate: SessionCandidate, + vendor: Vendor, + columns: AiRegistrySqlColumns, +): string => { + const pairs: Array = [] + + const authority = compileAuthoritySql(candidate, columns) + if (authority !== undefined) + pairs.push([`NOT (${authority})`, String(AI_SESSION_KEY_STATE.notAuthoritative)]) + + const presence = presenceExpr(candidate.key, "span", columns) + if (presence !== "1") pairs.push([`NOT ${presence}`, String(AI_SESSION_KEY_STATE.keyAbsent)]) + + const value = valueExpr(candidate.key, "span", columns) + const invalid: string[] = [] + if (candidate.validation.includes("non_empty")) invalid.push(`${value} = ''`) + if (candidate.validation.includes("not_in_decoy_values") && vendor.decoy_values.length > 0) + invalid.push( + `${value} IN (${vendor.decoy_values.map((decoy) => quoteSqlString(decoy.value)).join(", ")})`, + ) + if (invalid.length > 0) pairs.push([anyOf(invalid), String(AI_SESSION_KEY_STATE.keyInvalid)]) + + const resolved = String( + candidate.granularity === "session" ? AI_SESSION_KEY_STATE.session : AI_SESSION_KEY_STATE.subSession, + ) + return multiIf(pairs, resolved) +} + +/** `max` over candidates — a monotone quality order, so the reduction is order-independent. */ +const compileVendorStateSql = (vendor: Vendor, columns: AiRegistrySqlColumns): string => { + const states = vendor.session_candidates.map((candidate) => + compileCandidateStateSql(candidate, vendor, columns), + ) + if (states.length === 1) return states[0] as string + return `greatest(\n${states.map((state) => `\t${indent(state)}`).join(",\n")}\n)` +} + +/** + * The winning candidate's raw value: the first candidate, in declaration order, whose state equals + * the reduced state — which is exactly "hash from the candidate that produced the winning state, + * ties broken by candidate order". Empty below state 5, where no hash is written. + */ +const compileVendorKeyValueSql = ( + vendor: Vendor, + columns: AiRegistrySqlColumns, + sessionStateAlias: string, +): string => { + const pairs = vendor.session_candidates.map( + (candidate) => + [ + `${compileCandidateStateSql(candidate, vendor, columns)} = ${sessionStateAlias}`, + valueExpr(candidate.key, "span", columns), + ] as const, + ) + return `if(${sessionStateAlias} >= ${AI_SESSION_KEY_STATE.subSession}, ${multiIf(pairs, "''")}, '')` +} + +export const compileAiRegistrySql = ( + registry: AiRegistryDocument = aiRegistry, + options: CompileAiRegistrySqlOptions = {}, +): CompiledAiRegistrySql => { + const columns: AiRegistrySqlColumns = { ...DEFAULT_AI_REGISTRY_SQL_COLUMNS, ...options.columns } + const vendorAlias = options.vendorAlias ?? "AiVendorComputed" + const sessionStateAlias = options.sessionStateAlias ?? "AiSessionKeyStateComputed" + + const vendorExpr = multiIf( + collectVendorBranches(registry, columns).map( + (branch) => [branch.condition, quoteSqlString(branch.vendor)] as const, + ), + "''", + ) + + // Only vendors with candidates get a branch. Everything else falls through to "no vendor" (0) + // or "vendor has no session-key rules" (1) — the latter covers every `unknown:*` bucket. + const vendorsWithCandidates = registry.vendors.filter((vendor) => vendor.session_candidates.length > 0) + const noRulesFallback = `if(${vendorAlias} = '', ${AI_SESSION_KEY_STATE.notExamined}, ${AI_SESSION_KEY_STATE.noRules})` + + const sessionStateExpr = multiIf( + vendorsWithCandidates.map( + (vendor) => + [ + `${vendorAlias} = ${quoteSqlString(vendor.vendor)}`, + compileVendorStateSql(vendor, columns), + ] as const, + ), + noRulesFallback, + ) + + const sessionKeyValueExpr = multiIf( + vendorsWithCandidates.map( + (vendor) => + [ + `${vendorAlias} = ${quoteSqlString(vendor.vendor)}`, + compileVendorKeyValueSql(vendor, columns, sessionStateAlias), + ] as const, + ), + "''", + ) + + return { + vendorExpr, + sessionStateExpr, + sessionKeyValueExpr, + rulesVersion: registry.registry_version, + rulesVersionExpr: `toUInt32(${registry.registry_version})`, + columns, + vendorAlias, + sessionStateAlias, + } +} + +/** Output aliases of the two expressions that are not already named by `CompiledAiRegistrySql`. */ +export const AI_REGISTRY_SESSION_KEY_VALUE_ALIAS = "AiSessionKeyValueComputed" +export const AI_REGISTRY_RULES_VERSION_ALIAS = "AiRulesVersionComputed" + +export interface RenderAiRegistrySelectOptions { + /** + * Expressions projected alongside the four computed columns — `OrgId`, `TraceId`, `Timestamp` + * for a rebuild; `SpanId` for a differential test. They must exist on the source relation. + */ + readonly passthrough?: ReadonlyArray +} + +/** + * A complete, correctly layered SELECT over `from` (a table name, or a parenthesised subquery). + * + * Three nested projections, innermost first: vendor → session state → session key value. The + * layering is not cosmetic — see the module header on alias inlining. `SELECT *` carries the source + * columns inward so the outer expressions can still read `SpanAttributes`. + */ +export const renderAiRegistrySelect = ( + compiled: CompiledAiRegistrySql, + from: string, + options: RenderAiRegistrySelectOptions = {}, +): string => { + const projected = [ + ...(options.passthrough ?? []), + compiled.vendorAlias, + compiled.sessionStateAlias, + `${compiled.sessionKeyValueExpr} AS ${AI_REGISTRY_SESSION_KEY_VALUE_ALIAS}`, + `${compiled.rulesVersionExpr} AS ${AI_REGISTRY_RULES_VERSION_ALIAS}`, + ] + const withVendor = `SELECT\n\t*,\n\t${indent(compiled.vendorExpr)} AS ${compiled.vendorAlias}\nFROM ${from}` + const withState = `SELECT\n\t*,\n\t${indent(compiled.sessionStateExpr)} AS ${compiled.sessionStateAlias}\nFROM (\n\t${indent(withVendor)}\n)` + return `SELECT\n${projected.map((expression) => `\t${indent(expression)}`).join(",\n")}\nFROM (\n\t${indent(withState)}\n)` +} diff --git a/packages/domain/src/ai-registry/index.ts b/packages/domain/src/ai-registry/index.ts new file mode 100644 index 000000000..6923a87f9 --- /dev/null +++ b/packages/domain/src/ai-registry/index.ts @@ -0,0 +1,55 @@ +// Subpath barrel: `@maple/domain/ai-registry`. +// +// Deliberately NOT re-exported from the root `@maple/domain` barrel. `registry.json` is ~280 KB of +// vendored data, and web/cli import the root barrel — everything here is pure data plus string +// building, so it is safe to import anywhere, but nobody should pay for it implicitly. + +export { + AI_SESSION_KEY_STATE, + AI_SESSION_KEY_STATE_ELIGIBLE, + AiRegistryDocument, + AuthorityPredicate, + Matcher, + MatcherClass, + MatcherSignal, + PREDICATE_OPS, + PSEUDO_KEYS, + Predicate, + PseudoKeySchema, + SessionCandidate, + SessionGranularity, + UnknownTierRule, + ValidationToken, + Vendor, + aiRegistry, + aiRegistryVendorsBySlug, + type AiSessionKeyState, + type Caveat, + type DecoyKey, + type DecoyValue, + type PseudoKey, + type Variant, +} from "./schema" + +export { + PRIORITY_BANDS, + UNKNOWN_VENDOR_PREFIX, + formatRegistryViolations, + validateRegistry, + type RegistryViolation, + type RegistryViolationCode, +} from "./validate" + +export { + AI_REGISTRY_RULES_VERSION_ALIAS, + AI_REGISTRY_SESSION_KEY_VALUE_ALIAS, + DEFAULT_AI_REGISTRY_SQL_COLUMNS, + compileAiRegistrySql, + compilePredicateSql, + quoteSqlString, + renderAiRegistrySelect, + type AiRegistrySqlColumns, + type CompileAiRegistrySqlOptions, + type CompiledAiRegistrySql, + type RenderAiRegistrySelectOptions, +} from "./compile-sql" diff --git a/packages/domain/src/ai-registry/registry-fixtures.ts b/packages/domain/src/ai-registry/registry-fixtures.ts new file mode 100644 index 000000000..16610f9c2 --- /dev/null +++ b/packages/domain/src/ai-registry/registry-fixtures.ts @@ -0,0 +1,81 @@ +// Minimal hand-built registry documents for the validation and compiler tests. +// +// Built from scratch rather than cloned-and-mutated from `registry.json`: a negative test has to +// say out loud which single property it breaks, and a 280 KB clone with one field poked buries +// that. These fixtures are also the only place `value_prefix` appears at all — the real registry +// declares the operator (amendment D1) but no seed has yet needed it, so without a synthetic +// vendor that branch of the compiler would be untested. + +import type { AiRegistryDocument, Matcher, SessionCandidate, UnknownTierRule, Vendor } from "./schema" + +export const testMatcher = (overrides: Partial & Pick): Matcher => ({ + class: "attr", + sufficient: false, + source: "wire", + priority: 29_000, + ...overrides, +}) + +export const testCandidate = ( + overrides: Partial & Pick, +): SessionCandidate => ({ + authority_predicate: null, + validation: ["non_empty"], + granularity: "session", + verdict: "A", + ...overrides, +}) + +export const testVendor = (overrides: Partial & Pick): Vendor => ({ + matchers: [], + session_candidates: [], + decoy_keys: [], + decoy_values: [], + caveats: [], + ...overrides, +}) + +export const testRegistry = ( + vendors: ReadonlyArray, + unknownTier: ReadonlyArray = [], +): AiRegistryDocument => ({ + registry_version: 1, + generated_by: "test", + compiled_from: "test", + decisions: [], + algebra: { + ops: ["present", "eq", "key_prefix", "value_prefix"], + value_prefix_pseudo_keys: ["scope.name", "span.name", "scope.version", "scope.schema_url"], + canonicalization: "test", + }, + session_state_enum: { + "1": "vendor has no session-key rules", + "2": "span not session-authoritative", + "3": "authoritative, key absent", + "4": "key present, failed validation", + "5": "resolved at run/instance/user granularity", + "6": "resolved at session granularity", + reduction: "max over candidates", + }, + unknown_tier: unknownTier, + vendors, +}) + +/** + * A vendor exercising `value_prefix`, which the shipped registry declares but never uses. Models a + * hierarchical instrumentation-scope family: everything under `openinference.instrumentation.` is + * the same dialect, so the rule keys on the scope-name prefix rather than 20 exact names. + */ +export const valuePrefixVendor: Vendor = testVendor({ + vendor: "prefixed_vendor", + matchers: [ + testMatcher({ + class: "scope", + sufficient: true, + priority: 39_000, + owned_by: "library", + justification: "test fixture", + predicate: { op: "value_prefix", key: "scope.name", prefix: "openinference.instrumentation." }, + }), + ], +}) diff --git a/packages/domain/src/ai-registry/schema.ts b/packages/domain/src/ai-registry/schema.ts new file mode 100644 index 000000000..93db36d2c --- /dev/null +++ b/packages/domain/src/ai-registry/schema.ts @@ -0,0 +1,278 @@ +// Typed view of the vendored AI-vendor classification registry, decoded at module load. +// +// `registry.json` next to this file is a GENERATED artifact (see README.md) copied verbatim from +// the trace-capture repo. Nothing in this directory may hand-fix its data. This module's job is the +// opposite: describe the contract that both targets — the Rust detector in `apps/ingest` and the +// SQL compiler in `./compile-sql` — assume, and fail loudly at import time when a re-sync breaks it. +// +// Why Effect Schema rather than a hand-rolled type assertion: the artifact arrives as untrusted +// JSON from another repo, and the failure mode we care about is a *silent* shape drift (a matcher +// class the compiler skips, a granularity token nothing handles). A decode turns that into an +// import-time error, which is the same gate CI runs. +// +// Reserved-but-unused fields are modelled deliberately (`signal`, the `event`/`event_attr` matcher +// classes). Plan §1 reserves them in the schema for v1 and implements neither; leaving them out +// would make the first registry that uses them fail to decode instead of being ignored. + +import { Schema } from "effect" +import registryJson from "./registry.json" + +/** + * The four predicate operators of the restricted algebra (plan §1 + amendment D1). + * + * Deliberately closed: no negation, no conjunction, no span-event access, no JSON traversal. The + * restriction is what makes the Rust and SQL evaluators provably alignable (plan §6). + */ +export const PREDICATE_OPS = ["present", "eq", "key_prefix", "value_prefix"] as const + +/** + * `value_prefix` is restricted to these pseudo-keys, which are real columns on `traces` in SQL and + * plain byte compares in Rust. Allowing it over arbitrary attribute values would make the algebra + * unindexable (amendment D1). + */ +export const PSEUDO_KEYS = ["scope.name", "span.name", "scope.version", "scope.schema_url"] as const + +export type PseudoKey = (typeof PSEUDO_KEYS)[number] + +export const PseudoKeySchema = Schema.Literals(PSEUDO_KEYS) + +const PredicatePresent = Schema.Struct({ + op: Schema.Literal("present"), + key: Schema.String, +}) + +const PredicateEq = Schema.Struct({ + op: Schema.Literal("eq"), + key: Schema.String, + value: Schema.String, +}) + +const PredicateKeyPrefix = Schema.Struct({ + op: Schema.Literal("key_prefix"), + prefix: Schema.String, +}) + +const PredicateValuePrefix = Schema.Struct({ + op: Schema.Literal("value_prefix"), + key: PseudoKeySchema, + prefix: Schema.String, +}) + +/** One predicate of the restricted algebra. */ +export const Predicate = Schema.Union([ + PredicatePresent, + PredicateEq, + PredicateKeyPrefix, + PredicateValuePrefix, +]) +export type Predicate = Schema.Schema.Type + +/** + * Session-candidate authority predicates are the one place `any_of` appears. It is a bounded + * disjunction over the same algebra — not general conjunction — so both targets can still compile + * it to a flat OR. `null` means "every span of this vendor is authoritative". + */ +export const AuthorityPredicate = Schema.NullOr( + Schema.Union([Predicate, Schema.Struct({ any_of: Schema.Array(Predicate) })]), +) +export type AuthorityPredicate = Schema.Schema.Type + +/** + * Matcher classes, in plan §1's hoisting order. `event` / `event_attr` are reserved for the + * semconv migration of content into span events and are not implemented in v1 — the compiler + * rejects them rather than silently dropping them. + */ +export const MatcherClass = Schema.Literals(["resource", "scope", "attr", "event", "event_attr"]) +export type MatcherClass = Schema.Schema.Type + +/** Reserved on every matcher; only `traces` is produced today (plan §1). */ +export const MatcherSignal = Schema.Literals(["traces", "logs", "metrics"]) +export type MatcherSignal = Schema.Schema.Type + +export const Matcher = Schema.Struct({ + class: MatcherClass, + predicate: Predicate, + /** + * Unique across every matcher in the document. Sufficiency gates *participation*; this single + * global integer ranks the participants (plan §1). Bands are asserted in `./validate`. + */ + priority: Schema.Number.check(Schema.isInt()), + /** + * A sufficient match is an unconditional hit. An insufficient resource/scope match is only a + * conditional candidate, promoted at its own priority when the same span independently produces + * an attr-matcher hit for the same vendor. + */ + sufficient: Schema.Boolean, + source: Schema.Literals(["wire", "source_code"]), + /** Who owns the instrumentation scope. `generic`/`app` scopes may never be sufficient. */ + owned_by: Schema.optionalKey(Schema.NullOr(Schema.Literals(["library", "generic", "app"]))), + /** Required prose on sufficient resource matchers — see `./validate`. */ + justification: Schema.optionalKey(Schema.String), + signal: Schema.optionalKey(MatcherSignal), +}) +export type Matcher = Schema.Schema.Type + +/** Documented per-vendor choice; only `session` reaches state 6. */ +export const SessionGranularity = Schema.Literals(["session", "run", "user", "instance"]) +export type SessionGranularity = Schema.Schema.Type + +export const ValidationToken = Schema.Literals(["non_empty", "not_in_decoy_values"]) +export type ValidationToken = Schema.Schema.Type + +export const SessionCandidate = Schema.Struct({ + key: Schema.String, + authority_predicate: AuthorityPredicate, + validation: Schema.Array(ValidationToken), + granularity: SessionGranularity, + /** Seed-review confidence grade (A best). Carried for the education surface, not evaluated. */ + verdict: Schema.Literals(["A", "B", "C", "D"]), + source: Schema.optionalKey(Schema.Literals(["wire", "source_code"])), + note: Schema.optionalKey(Schema.String), +}) +export type SessionCandidate = Schema.Schema.Type + +/** Keys that look identifying but are not. Never consulted as a session key. */ +export const DecoyKey = Schema.Struct({ + key: Schema.String, + source: Schema.Literals(["wire", "source_code"]), + why: Schema.String, +}) +export type DecoyKey = Schema.Schema.Type + +/** Literal values that fail `not_in_decoy_values` validation (`"default"`, zero-UUIDs, …). */ +export const DecoyValue = Schema.Struct({ + value: Schema.String, + source: Schema.Literals(["wire", "source_code"]), + why: Schema.String, +}) +export type DecoyValue = Schema.Schema.Type + +export const Caveat = Schema.Union([Schema.String, Schema.Struct({ id: Schema.String, text: Schema.String })]) +export type Caveat = Schema.Schema.Type + +export const Variant = Schema.Struct({ + name: Schema.String, + trigger: Schema.String, + effects: Schema.String, + exercised_in_captures: Schema.Boolean, +}) +export type Variant = Schema.Schema.Type + +export const Vendor = Schema.Struct({ + /** Closed-set slug. `unknown:` is reserved for the fallback tier. */ + vendor: Schema.String, + matchers: Schema.Array(Matcher), + session_candidates: Schema.Array(SessionCandidate), + decoy_keys: Schema.Array(DecoyKey), + decoy_values: Schema.Array(DecoyValue), + caveats: Schema.Array(Caveat), + /** Path to the wire-verified seed in trace-capture. Absent for synthesized vendors. */ + seed: Schema.optionalKey(Schema.String), + goldens_status: Schema.optionalKey(Schema.Literals(["generated", "human_reviewed"])), + harness_keys_fixture_only: Schema.optionalKey(Schema.Array(Schema.Struct({ key: Schema.String }))), + variants: Schema.optionalKey(Schema.Array(Variant)), + /** Amendment D2: `langchain` carries `renamed_from: "langgraph"`. */ + renamed_from: Schema.optionalKey(Schema.String), + /** Amendment D3: `openinference-openai` has no seed of its own. */ + synthesized: Schema.optionalKey(Schema.Boolean), + justification: Schema.optionalKey(Schema.String), +}) +export type Vendor = Schema.Schema.Type + +/** + * Fallback fingerprints for spans that are recognisably AI telemetry but match no vendor. + * + * Plan §1 restricts `input.value` / `output.value` to firing only in co-occurrence with an + * OpenInference attribute. The algebra has no conjunction, so the compiler that produced this + * artifact honoured the restriction by *omitting* those fingerprints entirely — there is no + * co-occurrence rule to compile, and `./validate` asserts they never appear standalone. + */ +export const UnknownTierRule = Schema.Struct({ + bucket: Schema.String, + predicate: Predicate, + priority: Schema.Number.check(Schema.isInt()), + signal: Schema.optionalKey(MatcherSignal), +}) +export type UnknownTierRule = Schema.Schema.Type + +export const Algebra = Schema.Struct({ + ops: Schema.Array(Schema.String), + value_prefix_pseudo_keys: Schema.Array(Schema.String), + canonicalization: Schema.String, +}) + +/** + * The frozen state ladder. Values are append-only at v1: the discovery MV persists threshold + * comparisons over them (`state >= 3` is the eligibility contract), so renumbering would rewrite + * history (plan §2). + */ +export const AI_SESSION_KEY_STATE = { + /** Not examined, or no vendor. */ + notExamined: 0, + /** Vendor has no session-key rules (includes every `unknown:*` bucket). */ + noRules: 1, + /** Span is not session-authoritative. */ + notAuthoritative: 2, + /** Authoritative, key absent. */ + keyAbsent: 3, + /** Key present but failed validation (empty or a decoy value). */ + keyInvalid: 4, + /** Resolved at `run` / `instance` / `user` granularity. */ + subSession: 5, + /** Resolved at `session` granularity. */ + session: 6, +} as const + +export type AiSessionKeyState = (typeof AI_SESSION_KEY_STATE)[keyof typeof AI_SESSION_KEY_STATE] + +/** `state >= 3` — frozen, and the only threshold readers may hard-code (plan §2). */ +export const AI_SESSION_KEY_STATE_ELIGIBLE = AI_SESSION_KEY_STATE.keyAbsent + +const SessionStateEnum = Schema.Struct({ + "1": Schema.String, + "2": Schema.String, + "3": Schema.String, + "4": Schema.String, + "5": Schema.String, + "6": Schema.String, + reduction: Schema.String, +}) + +export const AiRegistryDocument = Schema.Struct({ + /** Global UInt32, bumped by any registry change; written to `traces.AiRulesVersion`. */ + registry_version: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)), + generated_by: Schema.String, + compiled_from: Schema.String, + decisions: Schema.Array(Schema.String), + algebra: Algebra, + session_state_enum: SessionStateEnum, + unknown_tier: Schema.Array(UnknownTierRule), + vendors: Schema.Array(Vendor), +}) +export type AiRegistryDocument = Schema.Schema.Type + +const decodeRegistry = Schema.decodeUnknownSync(AiRegistryDocument) + +/** + * Recursively frozen so a consumer that mutates the shared document gets a `TypeError` in + * development instead of poisoning every later compile in the same process. The registry is a + * process-wide singleton read once per batch (plan §1), never per span. + */ +const deepFreeze = (value: A): A => { + if (value === null || typeof value !== "object") return value + if (Object.isFrozen(value)) return value + Object.freeze(value) + for (const inner of Object.values(value as Record)) deepFreeze(inner) + return value +} + +/** + * The decoded, frozen registry. Decoding happens once at module load, so a shape drift introduced + * by a re-sync fails the import — and therefore CI — rather than the first classification. + */ +export const aiRegistry: AiRegistryDocument = deepFreeze(decodeRegistry(registryJson)) + +/** Convenience lookup; vendor slugs are unique (asserted in `./validate`). */ +export const aiRegistryVendorsBySlug: ReadonlyMap = new Map( + aiRegistry.vendors.map((vendor) => [vendor.vendor, vendor]), +) diff --git a/packages/domain/src/ai-registry/validate.test.ts b/packages/domain/src/ai-registry/validate.test.ts new file mode 100644 index 000000000..537b8fa72 --- /dev/null +++ b/packages/domain/src/ai-registry/validate.test.ts @@ -0,0 +1,294 @@ +// The registry gate. +// +// The first block runs every structural invariant against the real, vendored `registry.json`. It is +// what a future re-sync from trace-capture has to pass, and it is the reason the artifact can be +// treated as trusted input everywhere else in this package. +// +// The second block proves each rule actually fires. A validator that returns `[]` for everything +// also returns `[]` for a healthy registry, and that failure mode is invisible from the first block +// alone. + +import { describe, expect, it } from "vitest" +import { + PRIORITY_BANDS, + UNKNOWN_VENDOR_PREFIX, + formatRegistryViolations, + validateRegistry, + type RegistryViolationCode, +} from "./validate" +import { aiRegistry, type Predicate } from "./schema" +import { testCandidate, testMatcher, testRegistry, testVendor } from "./registry-fixtures" + +const codes = (violations: ReadonlyArray<{ code: RegistryViolationCode }>): Array => + violations.map((violation) => violation.code) + +describe("the vendored registry.json", () => { + it("decodes and satisfies every structural invariant", () => { + const violations = validateRegistry() + expect(formatRegistryViolations(violations)).toBe("") + expect(violations).toHaveLength(0) + }) + + it("has content, so a truncated re-sync cannot pass by being empty", () => { + expect(aiRegistry.vendors.length).toBeGreaterThan(15) + expect(aiRegistry.unknown_tier.length).toBeGreaterThan(0) + expect(aiRegistry.vendors.flatMap((vendor) => vendor.matchers).length).toBeGreaterThan(50) + }) + + it("gives every matcher a unique priority across the whole document", () => { + const priorities = [ + ...aiRegistry.vendors.flatMap((vendor) => vendor.matchers.map((m) => m.priority)), + ...aiRegistry.unknown_tier.map((rule) => rule.priority), + ] + expect(new Set(priorities).size).toBe(priorities.length) + }) + + it("keeps the D4 bands separated: sufficient > vendor-conditional > unknown tier", () => { + const sufficient = aiRegistry.vendors.flatMap((vendor) => + vendor.matchers.filter((m) => m.sufficient).map((m) => m.priority), + ) + const conditional = aiRegistry.vendors.flatMap((vendor) => + vendor.matchers.filter((m) => !m.sufficient).map((m) => m.priority), + ) + const unknown = aiRegistry.unknown_tier.map((rule) => rule.priority) + + expect(Math.min(...sufficient)).toBeGreaterThanOrEqual(PRIORITY_BANDS.sufficient.min) + expect(Math.min(...sufficient)).toBeGreaterThan(Math.max(...conditional)) + expect(Math.min(...conditional)).toBeGreaterThanOrEqual(PRIORITY_BANDS.vendorConditional.min) + expect(Math.min(...conditional)).toBeGreaterThan(Math.max(...unknown)) + expect(Math.min(...unknown)).toBeGreaterThanOrEqual(PRIORITY_BANDS.unknownTier.min) + expect(Math.max(...unknown)).toBeLessThanOrEqual(PRIORITY_BANDS.unknownTier.max) + }) + + it("reserves the unknown: namespace for the fallback tier only", () => { + for (const vendor of aiRegistry.vendors) + expect(vendor.vendor.startsWith(UNKNOWN_VENDOR_PREFIX)).toBe(false) + for (const rule of aiRegistry.unknown_tier) + expect(rule.bucket.startsWith(UNKNOWN_VENDOR_PREFIX)).toBe(true) + }) + + it("uses only the four algebra operators", () => { + const ops = new Set( + [ + ...aiRegistry.vendors.flatMap((vendor) => vendor.matchers.map((m) => m.predicate.op)), + ...aiRegistry.unknown_tier.map((rule) => rule.predicate.op), + ].map(String), + ) + expect([...ops].sort()).toEqual(["eq", "key_prefix", "present"]) + }) + + it("justifies every sufficient resource matcher", () => { + const sufficientResource = aiRegistry.vendors.flatMap((vendor) => + vendor.matchers.filter((m) => m.class === "resource" && m.sufficient), + ) + expect(sufficientResource.length).toBeGreaterThan(0) + for (const matcher of sufficientResource) + expect((matcher.justification ?? "").trim().length).toBeGreaterThan(0) + }) + + it("is byte-identical to the sha256 UPSTREAM.json pins", async () => { + // README.md says the artifact is generated and must never be hand-edited. This is that rule + // with teeth: a classification bug patched here instead of in trace-capture would silently + // diverge from the wire-verified seeds, and the next re-sync would drop the fix. + const { createHash } = await import("node:crypto") + const { readFile } = await import("node:fs/promises") + const here = new URL(".", import.meta.url) + const bytes = await readFile(new URL("registry.json", here)) + const upstream = JSON.parse(await readFile(new URL("UPSTREAM.json", here), "utf8")) as { + sha256: string + } + expect(createHash("sha256").update(bytes).digest("hex")).toBe(upstream.sha256) + }) + + it("never encodes input.value / output.value as standalone unknown-tier fingerprints", () => { + for (const rule of aiRegistry.unknown_tier) { + const key = "key" in rule.predicate ? rule.predicate.key : "" + expect(["input.value", "output.value"]).not.toContain(key) + } + }) +}) + +describe("validateRegistry rejects", () => { + it("duplicate priorities", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + matchers: [testMatcher({ priority: 29_001, predicate: { op: "present", key: "a.x" } })], + }), + testVendor({ + vendor: "b", + matchers: [testMatcher({ priority: 29_001, predicate: { op: "present", key: "b.x" } })], + }), + ]), + ) + expect(codes(violations)).toContain("duplicate_priority") + }) + + it("a vendor slug in the reserved unknown: namespace", () => { + const violations = validateRegistry(testRegistry([testVendor({ vendor: "unknown:mine" })])) + expect(codes(violations)).toContain("reserved_vendor_slug") + }) + + it("an unknown-tier bucket outside the reserved namespace", () => { + const violations = validateRegistry( + testRegistry( + [], + [{ bucket: "genai", predicate: { op: "present", key: "gen_ai.x" }, priority: 19_000 }], + ), + ) + expect(codes(violations)).toContain("unknown_bucket_not_reserved") + }) + + it("a sufficient matcher priced in the conditional band", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + matchers: [ + testMatcher({ + class: "scope", + sufficient: true, + priority: 21_000, + justification: "j", + predicate: { op: "eq", key: "scope.name", value: "a" }, + }), + ], + }), + ]), + ) + expect(codes(violations)).toContain("priority_out_of_band") + }) + + it("a sufficient resource matcher with no justification", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + matchers: [ + testMatcher({ + class: "resource", + sufficient: true, + priority: 39_000, + predicate: { op: "eq", key: "telemetry.sdk.name", value: "x" }, + }), + ], + }), + ]), + ) + expect(codes(violations)).toContain("missing_justification") + }) + + it("a sufficient scope matcher over an app-chosen scope", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + matchers: [ + testMatcher({ + class: "scope", + sufficient: true, + priority: 39_000, + owned_by: "generic", + justification: "j", + predicate: { op: "eq", key: "scope.name", value: "ai" }, + }), + ], + }), + ]), + ) + expect(codes(violations)).toContain("sufficient_generic_scope") + }) + + it("an insufficient scope matcher no attr matcher can promote", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + matchers: [ + testMatcher({ + class: "scope", + sufficient: false, + priority: 29_000, + owned_by: "generic", + justification: "j", + predicate: { op: "eq", key: "scope.name", value: "org.springframework.boot" }, + }), + ], + }), + ]), + ) + expect(codes(violations)).toContain("unpromotable_candidate") + }) + + it("value_prefix on a key that is not a pseudo-key", () => { + // The schema already makes this unrepresentable in TypeScript — hence the cast. The runtime + // check still has to exist: `validateRegistry` also runs over documents that reached it as + // plain JSON, which is the case a re-sync produces. + const badPredicate = { op: "value_prefix", key: "gen_ai.system", prefix: "x" } as unknown as Predicate + const violations = validateRegistry( + testRegistry([testVendor({ vendor: "a", matchers: [testMatcher({ predicate: badPredicate })] })]), + ) + expect(codes(violations)).toContain("value_prefix_not_pseudo_key") + }) + + it("a not_in_decoy_values token with no decoy values behind it", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + session_candidates: [ + testCandidate({ + key: "session.id", + validation: ["non_empty", "not_in_decoy_values"], + }), + ], + }), + ]), + ) + expect(codes(violations)).toContain("decoy_validation_without_values") + }) + + it("a key that is both a session candidate and a decoy key", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + session_candidates: [testCandidate({ key: "session.id" })], + decoy_keys: [{ key: "session.id", source: "wire", why: "why" }], + }), + ]), + ) + expect(codes(violations)).toContain("decoy_key_is_candidate") + }) + + it("a reserved-but-unimplemented matcher class", () => { + const violations = validateRegistry( + testRegistry([ + testVendor({ + vendor: "a", + matchers: [ + testMatcher({ class: "event", predicate: { op: "present", key: "gen_ai.choice" } }), + ], + }), + ]), + ) + expect(codes(violations)).toContain("unimplemented_matcher_class") + }) + + it("a standalone input.value fingerprint in the unknown tier", () => { + const violations = validateRegistry( + testRegistry( + [], + [ + { + bucket: "unknown:other", + predicate: { op: "present", key: "input.value" }, + priority: 19_000, + }, + ], + ), + ) + expect(codes(violations)).toContain("standalone_io_fingerprint") + }) +}) diff --git a/packages/domain/src/ai-registry/validate.ts b/packages/domain/src/ai-registry/validate.ts new file mode 100644 index 000000000..6d0e5e74c --- /dev/null +++ b/packages/domain/src/ai-registry/validate.ts @@ -0,0 +1,304 @@ +// Structural invariants of the AI-vendor registry that Effect Schema cannot express. +// +// The schema in `./schema` proves the artifact has the right *shape*. This module proves it has +// the right *semantics* — the properties both the Rust detector and the SQL compiler silently +// assume and neither would notice losing: +// +// - one global, unique, integer priority ranks every matcher (ties make resolution +// implementation-defined, so plan §1 makes them a CI failure); +// - the D4 priority bands hold, so band membership stays readable from the number; +// - the vendor slug set is closed and `unknown:` stays reserved for the fallback tier; +// - `value_prefix` never escapes its pseudo-keys (amendment D1); +// - a sufficient resource matcher — the claim that a whole *process* emits exactly one vendor — +// carries written justification. +// +// These run as Vitest assertions over the real `registry.json` (`./validate.test.ts`). That test is +// the gate a future registry re-sync has to pass. It reports; it never repairs. A violation means +// the upstream compiler in trace-capture produced a bad artifact and must be fixed there — patching +// the vendored JSON here would diverge it from the wire-verified seeds that justify every matcher. + +import { PREDICATE_OPS, PSEUDO_KEYS, aiRegistry } from "./schema" +import type { AiRegistryDocument, Matcher, Predicate } from "./schema" + +/** Slug namespace reserved for the unknown tier; no vendor may mint one. */ +export const UNKNOWN_VENDOR_PREFIX = "unknown:" + +/** + * Amendment D4's mechanical bands. + * + * `vendorConditional` covers attr matchers *and* insufficient resource/scope matchers. That is not + * a widening of D4: a conditional candidate is only ever promoted by an attr hit for the same + * vendor, so it has to rank in the same band as the matchers that promote it for the global + * priority order to mean anything. + */ +export const PRIORITY_BANDS = { + /** Sufficient scope/resource matchers — unconditional hits. */ + sufficient: { min: 30_000, max: 39_999 }, + /** Vendor attr matchers and insufficient (conditional) resource/scope matchers. */ + vendorConditional: { min: 20_000, max: 29_999 }, + /** Unknown-tier fingerprints — always last. */ + unknownTier: { min: 10_000, max: 19_999 }, +} as const + +export type RegistryViolationCode = + | "duplicate_priority" + | "priority_out_of_band" + | "band_overlap" + | "duplicate_vendor_slug" + | "reserved_vendor_slug" + | "unknown_bucket_not_reserved" + | "unsupported_op" + | "algebra_declaration_mismatch" + | "value_prefix_not_pseudo_key" + | "missing_justification" + | "sufficient_attr_matcher" + | "sufficient_generic_scope" + | "unpromotable_candidate" + | "unimplemented_matcher_class" + | "unimplemented_signal" + | "decoy_key_is_candidate" + | "decoy_validation_without_values" + | "standalone_io_fingerprint" + +export interface RegistryViolation { + readonly code: RegistryViolationCode + /** Where in the document: a vendor slug, `unknown_tier`, or `algebra`. */ + readonly where: string + readonly message: string +} + +const describePredicate = (predicate: Predicate): string => + predicate.op === "key_prefix" + ? `key_prefix(${predicate.prefix})` + : predicate.op === "present" + ? `present(${predicate.key})` + : predicate.op === "eq" + ? `eq(${predicate.key}, ${predicate.value})` + : `value_prefix(${predicate.key}, ${predicate.prefix})` + +const bandOf = (matcher: Matcher): { min: number; max: number } => + matcher.sufficient ? PRIORITY_BANDS.sufficient : PRIORITY_BANDS.vendorConditional + +/** + * Generic fingerprints that plan §1 permits only in co-occurrence with an OpenInference attribute. + * The algebra has no conjunction, so the only correct encoding is to omit them — a standalone rule + * on either key would fire on ordinary custom instrumentation. + */ +const CO_OCCURRENCE_ONLY_KEYS: ReadonlySet = new Set(["input.value", "output.value"]) + +/** + * Every structural check, over any registry document. Returns every violation rather than throwing + * on the first: a broken re-sync usually breaks several rules at once, and one message per run + * turns a five-minute fix into five CI rounds. + */ +export const validateRegistry = ( + registry: AiRegistryDocument = aiRegistry, +): ReadonlyArray => { + const violations: RegistryViolation[] = [] + const report = (code: RegistryViolationCode, where: string, message: string): void => { + violations.push({ code, where, message }) + } + + // --- algebra declaration matches what this package implements ------------------------------- + const declaredOps = [...registry.algebra.ops].sort() + if (declaredOps.join(",") !== [...PREDICATE_OPS].sort().join(",")) + report( + "algebra_declaration_mismatch", + "algebra", + `declares ops [${registry.algebra.ops.join(", ")}] but this package implements [${PREDICATE_OPS.join(", ")}]`, + ) + const declaredPseudo = [...registry.algebra.value_prefix_pseudo_keys].sort() + if (declaredPseudo.join(",") !== [...PSEUDO_KEYS].sort().join(",")) + report( + "algebra_declaration_mismatch", + "algebra", + `declares value_prefix pseudo-keys [${registry.algebra.value_prefix_pseudo_keys.join(", ")}] but this package implements [${PSEUDO_KEYS.join(", ")}]`, + ) + + // --- predicates ----------------------------------------------------------------------------- + const checkPredicate = (predicate: Predicate, where: string): void => { + if (!(PREDICATE_OPS as ReadonlyArray).includes(predicate.op)) + report("unsupported_op", where, `op '${predicate.op}' is outside the restricted algebra`) + // The schema already narrows `value_prefix` to pseudo-keys; re-checked here because this + // function also runs over registries built in tests, which bypass the decode. + if ( + predicate.op === "value_prefix" && + !(PSEUDO_KEYS as ReadonlyArray).includes(predicate.key) + ) + report( + "value_prefix_not_pseudo_key", + where, + `value_prefix on '${predicate.key}'; allowed only on ${PSEUDO_KEYS.join(", ")}`, + ) + } + + // --- priorities: unique, integral, banded ---------------------------------------------------- + const seenPriority = new Map() + const claimPriority = (priority: number, where: string): void => { + const previous = seenPriority.get(priority) + if (previous !== undefined) + report( + "duplicate_priority", + where, + `priority ${priority} is already used by ${previous}; priorities must be globally unique`, + ) + else seenPriority.set(priority, where) + } + + const slugs = new Set() + + for (const vendor of registry.vendors) { + const slug = vendor.vendor + if (slugs.has(slug)) report("duplicate_vendor_slug", slug, `vendor slug '${slug}' appears twice`) + slugs.add(slug) + if (slug.startsWith(UNKNOWN_VENDOR_PREFIX)) + report( + "reserved_vendor_slug", + slug, + `'${UNKNOWN_VENDOR_PREFIX}' is reserved for the unknown tier`, + ) + + const attrMatchers = vendor.matchers.filter((matcher) => matcher.class === "attr") + + for (const matcher of vendor.matchers) { + const where = `${slug}:${describePredicate(matcher.predicate)}` + checkPredicate(matcher.predicate, where) + claimPriority(matcher.priority, where) + + if (matcher.class === "event" || matcher.class === "event_attr") + report( + "unimplemented_matcher_class", + where, + `matcher class '${matcher.class}' is reserved in the schema but not implemented in v1`, + ) + if (matcher.signal !== undefined && matcher.signal !== "traces") + report( + "unimplemented_signal", + where, + `signal '${matcher.signal}' is reserved but only 'traces' is classified in v1`, + ) + + const band = bandOf(matcher) + if (matcher.priority < band.min || matcher.priority > band.max) + report( + "priority_out_of_band", + where, + `priority ${matcher.priority} is outside the ${matcher.sufficient ? "sufficient" : "vendor-conditional"} band [${band.min}, ${band.max}]`, + ) + + if (matcher.sufficient) { + if (matcher.class === "attr") + report( + "sufficient_attr_matcher", + where, + "sufficiency is a resource/scope concept; an attr matcher is always an unconditional hit at its own priority and must not be flagged sufficient", + ) + // A sufficient resource matcher claims the whole process emits exactly one vendor — + // true for a dedicated gateway, false for every application framework. Plan §1 makes + // the claim require prose. + if (matcher.class === "resource" && (matcher.justification ?? "").trim() === "") + report( + "missing_justification", + where, + "a sufficient resource matcher claims the whole process emits exactly one vendor and requires a written justification", + ) + if (matcher.owned_by === "generic" || matcher.owned_by === "app") + report( + "sufficient_generic_scope", + where, + `a scope owned_by '${matcher.owned_by}' is app-chosen and can never be sufficient`, + ) + } else if (matcher.class !== "attr" && attrMatchers.length === 0) { + // The promotion rule has nothing to promote it with, so the matcher can never + // contribute a hit — dead weight that reads like coverage. + report( + "unpromotable_candidate", + where, + `insufficient ${matcher.class} matcher, but '${slug}' declares no attr matcher that could promote it`, + ) + } + } + + // --- session candidates ------------------------------------------------------------------- + const decoyKeys = new Set(vendor.decoy_keys.map((decoy) => decoy.key)) + for (const candidate of vendor.session_candidates) { + const where = `${slug}:candidate(${candidate.key})` + if (decoyKeys.has(candidate.key)) + report( + "decoy_key_is_candidate", + where, + `'${candidate.key}' is listed both as a session candidate and as a decoy key`, + ) + if (candidate.validation.includes("not_in_decoy_values") && vendor.decoy_values.length === 0) + report( + "decoy_validation_without_values", + where, + "validation requires 'not_in_decoy_values' but the vendor declares no decoy_values, so the token is a no-op", + ) + const authority = candidate.authority_predicate + if (authority !== null) { + if ("any_of" in authority) + for (const predicate of authority.any_of) checkPredicate(predicate, where) + else checkPredicate(authority, where) + } + } + } + + // --- unknown tier --------------------------------------------------------------------------- + for (const rule of registry.unknown_tier) { + const where = `unknown_tier:${rule.bucket}:${describePredicate(rule.predicate)}` + checkPredicate(rule.predicate, where) + claimPriority(rule.priority, where) + if (!rule.bucket.startsWith(UNKNOWN_VENDOR_PREFIX)) + report( + "unknown_bucket_not_reserved", + where, + `unknown-tier bucket '${rule.bucket}' must be namespaced '${UNKNOWN_VENDOR_PREFIX}'`, + ) + if (rule.priority < PRIORITY_BANDS.unknownTier.min || rule.priority > PRIORITY_BANDS.unknownTier.max) + report( + "priority_out_of_band", + where, + `priority ${rule.priority} is outside the unknown-tier band [${PRIORITY_BANDS.unknownTier.min}, ${PRIORITY_BANDS.unknownTier.max}]`, + ) + if ( + (rule.predicate.op === "present" || rule.predicate.op === "eq") && + CO_OCCURRENCE_ONLY_KEYS.has(rule.predicate.key) + ) + report( + "standalone_io_fingerprint", + where, + `'${rule.predicate.key}' may only fire in co-occurrence with an OpenInference attribute; the algebra has no conjunction, so it must be omitted rather than encoded standalone`, + ) + } + + // --- the bands must not interleave in practice ---------------------------------------------- + const sufficientPriorities = registry.vendors.flatMap((vendor) => + vendor.matchers.filter((matcher) => matcher.sufficient).map((matcher) => matcher.priority), + ) + const conditionalPriorities = registry.vendors.flatMap((vendor) => + vendor.matchers.filter((matcher) => !matcher.sufficient).map((matcher) => matcher.priority), + ) + const unknownPriorities = registry.unknown_tier.map((rule) => rule.priority) + const strictlyAbove = (lower: ReadonlyArray, upper: ReadonlyArray): boolean => + lower.length === 0 || upper.length === 0 || Math.min(...upper) > Math.max(...lower) + + if (!strictlyAbove(conditionalPriorities, sufficientPriorities)) + report( + "band_overlap", + "priorities", + "a sufficient matcher does not outrank every conditional matcher; D4 requires sufficient > vendor-conditional", + ) + if (!strictlyAbove(unknownPriorities, conditionalPriorities)) + report( + "band_overlap", + "priorities", + "a vendor matcher does not outrank every unknown-tier fingerprint; D4 requires vendor-conditional > unknown tier", + ) + + return violations +} + +/** Renders violations for an assertion message. */ +export const formatRegistryViolations = (violations: ReadonlyArray): string => + violations.map((violation) => `[${violation.code}] ${violation.where}: ${violation.message}`).join("\n") diff --git a/packages/domain/tsconfig.json b/packages/domain/tsconfig.json index 37cc11429..84b973418 100644 --- a/packages/domain/tsconfig.json +++ b/packages/domain/tsconfig.json @@ -7,6 +7,8 @@ "types": ["node"], "moduleResolution": "bundler", "allowImportingTsExtensions": true, + // `src/ai-registry/registry.json` is a vendored artifact imported as data. + "resolveJsonModule": true, "verbatimModuleSyntax": true, "noEmit": true, "skipLibCheck": true,