From c9287c97e9c81496c4e2fba7d7fa928de8516403 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 13 Aug 2026 12:27:47 +0200 Subject: [PATCH 1/3] feat(domain): store AI classification columns on traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration 0015 adds five trailing columns to `traces` — vendor slug, session-key state, session-key hash, rules version, rollup hour — plus a `set(0)` skip index on the vendor and a token bloom filter on `ScopeName`, which the vendor rules match by prefix. Every column carries a DEFAULT, so the ALTER is metadata-only and rows written before the classifier existed still read back: `AiRulesVersion = 0` means "never examined", distinguishable from an examined-and-non-AI row. Nothing here materializes an index or column, and nothing mutates parts — the 30-day TTL retires the unindexed ones on its own. `requiredForIngest: true`, unlike the last two migrations: the gateway's INSERT now names all five columns, so a BYO-ClickHouse cluster that has not applied 0015 would reject every direct insert. Gating on it is the designed fallback — such an org resolves `clickhouse_ready = false` and routes to the managed pipeline until its schema syncs. The five columns declare snake_case JSONPaths rather than identity ones. That distinction is load-bearing and now also asserted: the insert-mapping generator drops a column that has a DEFAULT *and* an identity path, on the assumption the warehouse computes it. These are emitted on every span, so they must not match that shape. Co-Authored-By: Claude Fable 5 --- packages/domain/src/clickhouse/index.ts | 1 + .../0016_ai_classification_columns.ts | 76 +++++++++++++++++++ .../src/clickhouse/migrations/index.test.ts | 51 +++++++++++-- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 4 +- .../generated/tinybird-project-manifest.ts | 4 +- .../src/tinybird/datasources.contract.test.ts | 20 ++++- packages/domain/src/tinybird/datasources.ts | 47 ++++++++++++ 8 files changed, 191 insertions(+), 14 deletions(-) create mode 100644 packages/domain/src/clickhouse/migrations/0016_ai_classification_columns.ts diff --git a/packages/domain/src/clickhouse/index.ts b/packages/domain/src/clickhouse/index.ts index c37c5da9c..db47390dc 100644 --- a/packages/domain/src/clickhouse/index.ts +++ b/packages/domain/src/clickhouse/index.ts @@ -21,6 +21,7 @@ export { type MigrationStatement, } from "./migrations" export { performanceOnlySearchColumns } from "./migrations/0010_search_indexes" +export { AI_CLASSIFICATION_ALTER_STATEMENTS } from "./migrations/0016_ai_classification_columns" export { type BackfillSpec, isBackfill, diff --git a/packages/domain/src/clickhouse/migrations/0016_ai_classification_columns.ts b/packages/domain/src/clickhouse/migrations/0016_ai_classification_columns.ts new file mode 100644 index 000000000..5150e4192 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0016_ai_classification_columns.ts @@ -0,0 +1,76 @@ +/** + * Migration 0016 — AI classification columns on `traces`. + * + * Each column is appended at the end of the table and carries a constant + * DEFAULT, so the `ALTER` is metadata-only on ClickHouse and Tinybird alike and + * existing parts read the default for free. No `MATERIALIZE COLUMN` is needed + * for the same reason, and rows written before the classifier shipped stay + * readable (`AiRulesVersion = 0`). + * + * Column contracts (mirrored in `datasources.ts`, keep them in sync): + * - `AiVendor` '' means "not classified as AI", not "unknown vendor". + * - `AiRulesVersion` 0 means the row predates classification; any non-zero + * value means the span was examined, including a non-AI verdict. This is the + * only way to distinguish "not AI" from "never looked at". + * - `AiSessionKeyState` is a frozen quality enum 0-6. + * - `AiSessionKeyHash` is `cityHash64(value)`, 0 unless state >= 5. + * - `AiRollupHour` is a receive-time-clamped rollup hour written by ingest, + * epoch 0 until the ingest stage lands. + * + * `requiredForIngest: true` (it was `false` while the columns had no producer). + * The gateway now names all five in its INSERT column list, so a BYO cluster + * missing them would reject every direct insert — `clickHouseSchemaVersion` + * therefore has to become "16" so the routing gate catches that org first. + * + * What an unmigrated BYO org experiences: `fetch_ingest_key`'s + * `SCHEMA_REVISION_COMPATIBLE_SQL` compares the org's stamped + * `org_clickhouse_settings.schema_version` against this value numerically, so a + * cluster still below 16 resolves `clickhouse_ready = false` and its traffic + * routes to the managed pipeline instead of its own cluster. No data is lost and + * no insert fails; the org silently falls back until `applySchema` stamps 16, at + * which point routing returns on the next 30s cache TTL. This is the designed + * mechanism, not a side effect — contrast migrations 0010/0014/0015, which + * changed nothing the gateway writes and so stayed `requiredForIngest: false`. + * + * The two skip indexes are declared on the datasource as well — that is what + * puts them on managed (Tinybird) orgs and freshly bootstrapped clusters; these + * statements backfill clusters already at version 15. Deliberately no + * `MATERIALIZE INDEX`: that is a mutation over the whole table, and the 30-day + * TTL rolls every unindexed part out on its own. + * + * Both index types are ClickHouse 24.12-compatible. `set(0)` is unbounded on + * purpose (the vendor allowlist is closed at ~30 values, and a capped set that + * overflows silently degrades to always-match); `ScopeName` gets tokenbf_v1 + * rather than bloom_filter because the registry's scope matchers include prefix + * rules. + */ +/** + * The ALTER list itself, exported because it has a second execution site: the + * CLI's local-store v5 -> v6 module runs exactly these statements against chDB + * (`apps/cli/src/server/local-store-migrations/v5-to-v6-ai-classification-columns.ts`). + * That store's `traces` already exists, so bootstrapping the v6 DDL is a no-op on + * it and the columns and indexes arrive only through these ALTERs. + * + * One definition rather than two copies and a "keep these in sync" comment — + * same reasoning as `SERVICE_AI_VENDORS_HOURLY_SELECT_SQL`, which 0017 and the + * Tinybird materialization share. Retuning `idx_scope_name` here used to leave + * every migrated local store on the old index with nothing failing, because + * v5 -> v6's `verify` compares against the *frozen* v6 manifest, which records + * the index by name and not by parameters. + */ +export const AI_CLASSIFICATION_ALTER_STATEMENTS = [ + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiVendor LowCardinality(String) DEFAULT ''", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiSessionKeyState UInt8 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiSessionKeyHash UInt64 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiRulesVersion UInt32 DEFAULT 0", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS AiRollupHour DateTime('UTC') DEFAULT toDateTime(0)", + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4", + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4", +] as const + +export const migration_0016_ai_classification_columns = { + version: 16, + description: "Add AI classification columns and vendor/scope skip indexes to traces", + requiredForIngest: true, + statements: AI_CLASSIFICATION_ALTER_STATEMENTS, +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index f0a215089..db441032f 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -21,6 +21,7 @@ import { migration_0015_service_overview_minutely, serviceOverviewMinutelyBackfill, } from "./0015_service_overview_minutely" +import { migration_0016_ai_classification_columns } from "./0016_ai_classification_columns" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -35,17 +36,27 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) - expect(migrations.at(-1)).toBe(migration_0015_service_overview_minutely) - expect(latestMigrationVersion).toBe(15) - // 0010, 0014 and 0015 are performance-only, so the ingest-gating version - // skips all three and stays at 13 — nothing writes `web_events` or - // `service_overview_minutely` directly, and bumping it would un-ready every - // BYO-CH org's ingest routing for a read-path change. - expect(clickHouseSchemaVersion).toBe("13") + expect(migrations.map((m) => m.version)).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ]) + expect(migrations.at(-1)).toBe(migration_0016_ai_classification_columns) + expect(latestMigrationVersion).toBe(16) + // 0010, 0014 and 0015 are performance/storage-only, so the ingest-gating + // version skips all three — nothing writes `web_events` or + // `service_overview_minutely` directly and search indexes change nothing the + // gateway sends, and bumping for any of them would un-ready every BYO-CH + // org's ingest routing for a read-path change. expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) expect(migration_0015_service_overview_minutely.requiredForIngest).toBe(false) + + // 0016 is different: the gateway's INSERT now names all five AI columns, + // so a BYO cluster without them would reject every direct insert. Gating + // on it is the designed fallback — an org stamped below 16 resolves + // `clickhouse_ready = false` and routes to the managed pipeline until its + // schema syncs. + expect(migration_0016_ai_classification_columns.requiredForIngest).toBe(true) + expect(clickHouseSchemaVersion).toBe("16") }) it("installs service_overview_minutely with a live-write MV and no POPULATE", () => { @@ -117,6 +128,30 @@ describe("ClickHouse migrations", () => { ) }) + it("adds the AI classification columns as defaulted trailing columns with no mutation", () => { + const sql = migration_0016_ai_classification_columns.statements.join("\n") + + // Every column defaulted: that is what makes the ALTER metadata-only and + // keeps rows written before the classifier shipped readable + // (`AiRulesVersion = 0` = never examined). + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiVendor LowCardinality(String) DEFAULT ''") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiSessionKeyState UInt8 DEFAULT 0") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiSessionKeyHash UInt64 DEFAULT 0") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiRulesVersion UInt32 DEFAULT 0") + expect(sql).toContain("ADD COLUMN IF NOT EXISTS AiRollupHour DateTime('UTC') DEFAULT toDateTime(0)") + expect(sql).toContain("ADD INDEX IF NOT EXISTS idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4") + expect(sql).toContain( + "ADD INDEX IF NOT EXISTS idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4", + ) + + // Nothing here may rewrite parts: the 30-day TTL retires unindexed parts on + // its own, and a whole-table mutation on `traces` is the expensive mistake. + expect(sql).not.toContain("MATERIALIZE INDEX") + expect(sql).not.toContain("MATERIALIZE COLUMN") + expect(sql).not.toContain("OPTIMIZE TABLE") + expect(migration_0016_ai_classification_columns.statements.filter(isBackfill)).toHaveLength(0) + }) + it("installs web_events with a live-write MV and no POPULATE", () => { const sql = migration_0014_web_events.statements.filter((stmt) => !isBackfill(stmt)).join("\n") diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 3c65b6337..1e90d9786 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -14,6 +14,7 @@ import { migration_0012_session_event_attribute_keys } from "./0012_session_even import { migration_0013_service_map_ingest_bridge } from "./0013_service_map_ingest_bridge" import { migration_0014_web_events } from "./0014_web_events" import { migration_0015_service_overview_minutely } from "./0015_service_overview_minutely" +import { migration_0016_ai_classification_columns } from "./0016_ai_classification_columns" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -60,6 +61,7 @@ export const migrations: ReadonlyArray = [ migration_0013_service_map_ingest_bridge, migration_0014_web_events, migration_0015_service_overview_minutely, + migration_0016_ai_classification_columns, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index eab3088b7..666d55e70 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7" as const +export const projectRevision = "3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -39,7 +39,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS trace_detail_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, SpanId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS trace_list_mv (\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL Timestamp + INTERVAL 30 DAY", - "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n AiVendor LowCardinality(String) DEFAULT '',\n AiSessionKeyState UInt8 DEFAULT 0,\n AiSessionKeyHash UInt64 DEFAULT 0,\n AiRulesVersion UInt32 DEFAULT 0,\n AiRollupHour DateTime('UTC') DEFAULT toDateTime(0),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4,\n INDEX idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS web_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 42aaa991d..7e9a26e63 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7" as const +export const projectRevision = "3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349" as const export const datasources = [ { @@ -182,7 +182,7 @@ export const datasources = [ { name: "traces", content: - "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) `json:$.ResourceAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) `json:$.ScopeAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) `json:$.SpanAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes))\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) `json:$.ResourceAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) `json:$.ScopeAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) `json:$.SpanAttributeItems[:]` DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n AiVendor LowCardinality(String) `json:$.ai_vendor` DEFAULT '',\n AiSessionKeyState UInt8 `json:$.ai_session_key_state` DEFAULT 0,\n AiSessionKeyHash UInt64 `json:$.ai_session_key_hash` DEFAULT 0,\n AiRulesVersion UInt32 `json:$.ai_rules_version` DEFAULT 0,\n AiRollupHour DateTime('UTC') `json:$.ai_rollup_hour` DEFAULT toDateTime(0)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4\n idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4", }, { name: "traces_aggregates_hourly", diff --git a/packages/domain/src/tinybird/datasources.contract.test.ts b/packages/domain/src/tinybird/datasources.contract.test.ts index 11cd48566..7f06f10fc 100644 --- a/packages/domain/src/tinybird/datasources.contract.test.ts +++ b/packages/domain/src/tinybird/datasources.contract.test.ts @@ -69,6 +69,13 @@ const EXPECTED_TOPLEVEL_KEYS = { "links_span_id", "links_trace_state", "links_attributes", + // AI classification, emitted on every span by the ingest row builder. + // `ai_rollup_hour` is written whether or not classification is enabled. + "ai_vendor", + "ai_session_key_state", + "ai_session_key_hash", + "ai_rules_version", + "ai_rollup_hour", ]), metrics_sum: new Set([...metricCommonKeys(), "value", "aggregation_temporality", "is_monotonic"]), metrics_gauge: new Set([...metricCommonKeys(), "value"]), @@ -132,7 +139,7 @@ function topLevelKey(jsonPath: string): string | null { function emittedTopLevelKeys(datasource: DatasourceDefinition): Set { const keys = new Set() - for (const column of Object.values(datasource.options.schema)) { + for (const [name, column] of Object.entries(datasource.options.schema)) { const defaultExpression = ( column as { readonly type?: { @@ -140,9 +147,18 @@ function emittedTopLevelKeys(datasource: DatasourceDefinition): Set { } } ).type?.modifiers?.defaultExpression - if (defaultExpression !== undefined) continue const path = getColumnJsonPath(column) if (!path) continue + // The same rule `scripts/generate-clickhouse-insert-mappings.ts` applies: + // a column is treated as warehouse-computed — and so not something the + // gateway sends — only when it has a DEFAULT *expression* **and** an + // identity JSONPath (`$.`), which is what a column with no + // declared path falls back to. A DEFAULT expression paired with an + // explicitly declared path (`AiRollupHour` → `$.ai_rollup_hour`) is a + // column the gateway does emit, and keeps its DEFAULT only for rows + // written before it existed. + const isComputedIdentityPath = path === `$.${name}` || path === `$.${name}[:]` + if (defaultExpression !== undefined && isComputedIdentityPath) continue const top = topLevelKey(path) if (top) keys.add(top) } diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index a219d36c4..3471c72e0 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -247,6 +247,36 @@ export const traces = defineDatasource("traces", { SpanAttributeItems: column(t.array(t.string()).defaultExpr(attributeItemsExpr("SpanAttributes")), { jsonPath: "$.SpanAttributeItems[:]", }), + /** + * AI classification verdict, written by the ingest classifier. + * + * Each column keeps its constant DEFAULT — rows written before the + * classifier shipped, and any writer that predates it, still read back + * correctly — but now also declares a **snake_case** JSONPath, matching + * the gateway-emitted columns above rather than the identity paths used + * by the computed-DEFAULT columns (`SampleRate`, `IsEntryPoint`, the + * `*AttributeItems`). That distinction is load-bearing: the insert-mapping + * generator drops a column when it has a DEFAULT *and* an identity + * JSONPath, on the assumption the gateway never emits it. These five are + * emitted on every span, so they must not match that shape. + */ + /** Normalized vendor slug from the closed allowlist. '' = not classified as AI. */ + AiVendor: column(t.string().lowCardinality().default(""), { jsonPath: "$.ai_vendor" }), + /** Frozen session-key quality enum, 0-6. Higher = stronger provenance. */ + AiSessionKeyState: column(t.uint8().default(0), { jsonPath: "$.ai_session_key_state" }), + /** cityHash64 of the session-key value. 0 unless AiSessionKeyState >= 5. */ + AiSessionKeyHash: column(t.uint64().default(0), { jsonPath: "$.ai_session_key_hash" }), + /** Classifier rule-set version. 0 = row predates classification; non-zero means + * the span was examined, including a non-AI verdict. */ + AiRulesVersion: column(t.uint32().default(0), { jsonPath: "$.ai_rules_version" }), + /** + * Receive-time-clamped rollup hour. Written unconditionally by the row + * builder for every span, classification flag on or off, as + * `YYYY-MM-DD HH:MM:SS` — no row may carry a garbage rollup hour. + */ + AiRollupHour: column(t.dateTime("UTC").defaultExpr("toDateTime(0)"), { + jsonPath: "$.ai_rollup_hour", + }), }, indexes: [ { @@ -291,6 +321,23 @@ export const traces = defineDatasource("traces", { type: "bloom_filter(0.01)", granularity: 1, }, + { + // set(0) is deliberately unbounded: the vendor domain is a closed + // ~30-value allowlist, and a capped set() that overflows silently + // degrades to always-match rather than erroring. + name: "idx_ai_vendor", + expr: "AiVendor", + type: "set(0)", + granularity: 4, + }, + { + // tokenbf rather than bloom_filter because registry scope matchers + // include prefix rules, which need token-level lookups. + name: "idx_scope_name", + expr: "ScopeName", + type: "tokenbf_v1(4096, 3, 0)", + granularity: 4, + }, ], engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", From 946b35c5fd542d9d46a4603ae5cdb01500ed707a Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 13 Aug 2026 12:28:08 +0200 Subject: [PATCH 2/3] feat(ingest): write AI classification columns on the ingest path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The row builder now classifies each span and stamps the five columns. Inputs are built once per accepted payload, not per span: the migration-window flag is read once and the batch receive time is captured once, so every span in one payload clamps against the same instant. `AiRollupHour` is written unconditionally, flag on or off. It is the rollup's partition key and the span timestamp is attacker- and replay-controlled, so it is clamped at write time to `[receive - 7d, receive + 1d]`. Clamping in the view instead would need `now()`, which a later partition rebuild re-evaluates and which would silently relocate rows across hours. On the attribute-mapping path the classifier reads a first-occurrence-wins view of the wire attributes rather than the row's stored Map, which keeps last-wins canonicalization. The two rules only disagree on a span carrying a duplicate rule-key, and the verdict must not depend on whether the org happens to have mapping rules configured. Observability is batch-level, never per span — a span per classification on this path is what the self-observability rule forbids. The accept span carries whether the flag was on and how many spans were examined; `ingest_ai_spans_examined_total` is labeled by signal only, exactly like `native_rows`, so the two series are directly comparable and any divergence is a bug. Also here: - An adversarial fixture module driving `encode_traces` end to end, with a reproducibility check and a branch-coverage check over the written rows. - A ClickHouse E2E pinning `AiSessionKeyHash` to `cityHash64`. Without it a divergence returns zero rows and puts a permanent discontinuity in a 400-day-TTL sketch, with nothing else failing — so CI runs it, and the ClickHouse job's path filter now also watches the CityHash port. - A schema probe asserting the live `traces` columns against the generated schema. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 23 + ...rehouseQueryService.clickhouse.e2e.test.ts | 86 + apps/ingest/benches/ingest_bench.rs | 6 +- .../adversarial/adversarial-spans.jsonl | 348 ++ apps/ingest/src/ai_adversarial_fixtures.rs | 3033 +++++++++++++++++ apps/ingest/src/clickhouse_insert_mappings.rs | 10 +- apps/ingest/src/main.rs | 239 +- apps/ingest/src/metrics.rs | 19 + apps/ingest/src/otel.rs | 6 + apps/ingest/src/telemetry.rs | 693 +++- .../ai/hash-alignment.clickhouse.e2e.test.ts | 176 + 11 files changed, 4591 insertions(+), 48 deletions(-) create mode 100644 apps/ingest/fixtures/adversarial/adversarial-spans.jsonl create mode 100644 apps/ingest/src/ai_adversarial_fixtures.rs create mode 100644 packages/domain/src/ai/hash-alignment.clickhouse.e2e.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a0d9b0ea..c503f27fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,12 @@ jobs: - 'packages/domain/src/clickhouse/**' - 'packages/domain/src/tinybird/**' - 'packages/domain/src/generated/**' + # The hash-contract E2E lives here (not in apps/api), and + # the thing it pins — the in-crate CityHash 1.0.2 port — + # is the one input this job cannot see change, so the + # ingest source is a trigger too. + - 'packages/domain/src/ai/**' + - 'apps/ingest/src/cityhash102.rs' postgres: - *base # The connection layer and everything that decides how many @@ -542,6 +548,23 @@ jobs: bun run --filter=@maple/api test -- src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts + # `--filter=@maple/domain`, not `@maple/api`: this suite lives in + # packages/domain and was therefore skipped by every other step here. + # It pins `AiSessionKeyHash` = ClickHouse `cityHash64`, which is what + # keeps the read path's `WHERE AiSessionKeyHash = cityHash64({id})` + # and the column's repairability from `SpanAttributes` true. A + # divergence returns zero rows and puts a permanent discontinuity in + # a 400-day-TTL sketch, with nothing else failing. + - name: Verify the AI session-key hash contract + env: + CLICKHOUSE_E2E: "1" + CLICKHOUSE_E2E_URL: http://127.0.0.1:8123 + CLICKHOUSE_E2E_USER: maple + CLICKHOUSE_E2E_PASSWORD: maple + run: >- + bun run --filter=@maple/domain test -- + src/ai/hash-alignment.clickhouse.e2e.test.ts + local-checkpoint-native: name: Local checkpoint native needs: changes diff --git a/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts index 943ac50e6..9b9d5e4c4 100644 --- a/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts @@ -23,6 +23,9 @@ import { const enabled = clickhouseE2eEnabled const database = uniqueDatabase("maple_raw_sql_e2e") const orgId = "org_raw_sql_e2e" +/** Isolates the migration-0016 default-readback probe row from the row-level + * fixtures the query tests assert on. */ +const aiProbeOrgId = "org_raw_sql_e2e_ai_probe" const assertSearchSchemaApplied = async (): Promise => { const migrationRevision = ( @@ -141,6 +144,88 @@ SETTINGS enable_full_text_index = 1`, assert.include(explain, "idx_lower_body_text") } +/** + * Migration 0016 is storage-only — nothing writes these columns yet — so the + * only thing that can prove it applied is the physical schema. Asserted against + * a real server because `set(0)` and `tokenbf_v1` are the kind of DDL a + * SQL-text test happily accepts and ClickHouse rejects. + */ +const assertAiClassificationSchemaApplied = async (): Promise => { + const migrationRevision = ( + await clickhouseExec( + "SELECT count() FROM _maple_schema_migrations WHERE version = 15 FORMAT TabSeparated", + database, + ) + ).trim() + assert.strictEqual(migrationRevision, "1", "AI classification migration 15 was not recorded") + + const columns = ( + await clickhouseExec( + `SELECT name, type, default_kind +FROM system.columns +WHERE database = currentDatabase() + AND table = 'traces' + AND name IN ('AiVendor', 'AiSessionKeyState', 'AiSessionKeyHash', 'AiRulesVersion', 'AiRollupHour') +ORDER BY name +FORMAT TabSeparated`, + database, + ) + ) + .trim() + // TabSeparated escapes single quotes, so `DateTime('UTC')` arrives as + // `DateTime(\'UTC\')`. Unescape rather than encode the escaping into the + // expectation, which would read as a typo. + .replaceAll("\\'", "'") + assert.strictEqual( + columns, + [ + "AiRollupHour\tDateTime('UTC')\tDEFAULT", + "AiRulesVersion\tUInt32\tDEFAULT", + "AiSessionKeyHash\tUInt64\tDEFAULT", + "AiSessionKeyState\tUInt8\tDEFAULT", + "AiVendor\tLowCardinality(String)\tDEFAULT", + ].join("\n"), + "AI classification columns are missing, mistyped, or not default-computed", + ) + + const indexes = ( + await clickhouseExec( + `SELECT name, type_full, granularity +FROM system.data_skipping_indices +WHERE database = currentDatabase() + AND table = 'traces' + AND name IN ('idx_ai_vendor', 'idx_scope_name') +ORDER BY name +FORMAT TabSeparated`, + database, + ) + ).trim() + assert.strictEqual( + indexes, + ["idx_ai_vendor\tset(0)\t4", "idx_scope_name\ttokenbf_v1(4096, 3, 0)\t4"].join("\n"), + "AI classification skip indexes are missing or declared with the wrong type", + ) + + // A writer that names none of the new columns — which is every writer until + // the classifier ships — must still land a readable row. Written under its own + // org so it stays out of the row-level fixtures the tests below assert on. + await clickhouseExec( + `INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode) + VALUES ('${aiProbeOrgId}', now64(9), 'trace-ai-default', 'span-ai-default', 'GET /ai', 'Server', 'api', 1, 'Ok')`, + database, + ) + const defaults = ( + await clickhouseExec( + `SELECT AiVendor = '', AiSessionKeyState, AiSessionKeyHash, AiRulesVersion, toUnixTimestamp(AiRollupHour) +FROM traces +WHERE OrgId = '${aiProbeOrgId}' AND TraceId = 'trace-ai-default' +FORMAT TabSeparated`, + database, + ) + ).trim() + assert.strictEqual(defaults, "1\t0\t0\t0\t0", "AI classification defaults do not read back") +} + const trackedDbs: TestDb[] = [] const asOrgId = Schema.decodeUnknownSync(OrgId) const asUserId = Schema.decodeUnknownSync(UserId) @@ -198,6 +283,7 @@ describe.skipIf(!enabled)("WarehouseQueryService ClickHouse raw-SQL E2E", () => await clickhouseExec(`CREATE DATABASE ${database}`) await applyRealMigrations(database) await assertSearchSchemaApplied() + await assertAiClassificationSchemaApplied() await clickhouseExec( `INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode) diff --git a/apps/ingest/benches/ingest_bench.rs b/apps/ingest/benches/ingest_bench.rs index fd5b99675..9faf942bd 100644 --- a/apps/ingest/benches/ingest_bench.rs +++ b/apps/ingest/benches/ingest_bench.rs @@ -15,7 +15,8 @@ use flate2::read::GzDecoder; use maple_ingest::ai_classifier::ResourceContext; use maple_ingest::ai_registry::registry; use maple_ingest::telemetry::{ - ClickHouseBreakerConfig, DatasourceNames, SamplingPolicy, TelemetryPipeline, TinybirdConfig, + AiClassificationSettings, ClickHouseBreakerConfig, DatasourceNames, SamplingPolicy, + TelemetryPipeline, TinybirdConfig, }; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}; @@ -70,6 +71,9 @@ fn bench_ingest_accept(c: &mut Criterion) { black_box(&fixture.traces), &SamplingPolicy::default(), &[], + // Classification's own cost is measured by the + // `ai_classifier` group; this one measures WAL ack. + &AiClassificationSettings::disabled(), ) .await .expect("accept traces"), diff --git a/apps/ingest/fixtures/adversarial/adversarial-spans.jsonl b/apps/ingest/fixtures/adversarial/adversarial-spans.jsonl new file mode 100644 index 000000000..3daeb04ff --- /dev/null +++ b/apps/ingest/fixtures/adversarial/adversarial-spans.jsonl @@ -0,0 +1,348 @@ +{"category":"resolution/sufficient_scope","id":"sufficient_scope/agno/0","note":"sufficient scope matcher for agno, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"agno"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/agno/0","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"agno"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/claude_agent_sdk/1","note":"sufficient scope matcher for claude_agent_sdk, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/claude_agent_sdk/1","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/claude_agent_sdk/2","note":"sufficient scope matcher for claude_agent_sdk, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/claude_agent_sdk/2","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/claude_agent_sdk/3","note":"sufficient scope matcher for claude_agent_sdk, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/claude_agent_sdk/3","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/crewai/4","note":"sufficient scope matcher for crewai, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"crewai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/crewai/4","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"crewai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/crewai/5","note":"sufficient scope matcher for crewai, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"crewai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/crewai/5","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"crewai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/dspy/6","note":"sufficient scope matcher for dspy, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"dspy"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/dspy/6","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"dspy"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/flue/7","note":"sufficient scope matcher for flue, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"flue"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/flue/7","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"flue"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/google_adk/8","note":"sufficient scope matcher for google_adk, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"google_adk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/google_adk/8","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"google_adk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/haystack/9","note":"sufficient scope matcher for haystack, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"haystack"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/haystack/9","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"haystack"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/langchain/10","note":"sufficient scope matcher for langchain, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"langchain"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/langchain/10","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"langchain"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/litellm/11","note":"sufficient scope matcher for litellm, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"litellm"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/litellm/11","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"litellm"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/llamaindex/12","note":"sufficient scope matcher for llamaindex, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"llamaindex"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/llamaindex/12","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"llamaindex"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/mastra/13","note":"sufficient scope matcher for mastra, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"mastra"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/mastra/13","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"mastra"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/microsoft_agent_framework/14","note":"sufficient scope matcher for microsoft_agent_framework, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/microsoft_agent_framework/14","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/openai_agents_sdk/15","note":"sufficient scope matcher for openai_agents_sdk, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"openai_agents_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/openai_agents_sdk/15","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"openai_agents_sdk"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/openinference-openai/16","note":"sufficient scope matcher for openinference-openai, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"openinference-openai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/openinference-openai/16","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"openinference-openai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/pydantic_ai/17","note":"sufficient scope matcher for pydantic_ai, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"pydantic_ai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/pydantic_ai/17","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"pydantic_ai"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/semantic_kernel/18","note":"sufficient scope matcher for semantic_kernel, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/semantic_kernel/18","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/semantic_kernel/19","note":"sufficient scope matcher for semantic_kernel, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/semantic_kernel/19","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/semantic_kernel/20","note":"sufficient scope matcher for semantic_kernel, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/semantic_kernel/20","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/semantic_kernel/21","note":"sufficient scope matcher for semantic_kernel, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/semantic_kernel/21","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/semantic_kernel/22","note":"sufficient scope matcher for semantic_kernel, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/semantic_kernel/22","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/smolagents/23","note":"sufficient scope matcher for smolagents, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"smolagents"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/smolagents/23","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"smolagents"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/strands/24","note":"sufficient scope matcher for strands, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"strands"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/strands/24","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"strands"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/semantic_kernel/25","note":"sufficient scope matcher for semantic_kernel, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/semantic_kernel/25","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope/semantic_kernel/26","note":"sufficient scope matcher for semantic_kernel, no AI attribute at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/sufficient_scope","id":"sufficient_scope_hoisted/semantic_kernel/26","note":"second span under the same hoisted scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/insufficient_promoted","id":"promoted/spring_ai_scope","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"spring_ai"}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/spring_ai_scope","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_promoted","id":"promoted/vercel_scope_ai","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"vercel_ai_sdk"}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/vercel_scope_ai","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_promoted","id":"promoted/vercel_scope_gen_ai","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"vercel_ai_sdk"}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/vercel_scope_gen_ai","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_promoted","id":"promoted/claude_resource","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"claude_agent_sdk"}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/claude_resource","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_promoted","id":"promoted/litellm_resource","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/litellm_resource","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_promoted","id":"promoted/llamaindex_resource","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"17391958930642725075","session_key_hex":"72756e2d31","session_state":5,"vendor":"llamaindex"}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/llamaindex_resource","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_promoted","id":"promoted/langchain_resource","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"langchain"}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/langchain_resource","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/insufficient_promoted","id":"promoted/effect_ai_resource","note":"insufficient resource/scope candidate promoted by a same-vendor hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"effect_ai"}} +{"category":"resolution/insufficient_not_promoted","id":"not_promoted/effect_ai_resource","note":"same insufficient evidence with nothing to promote it","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/agno","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"agno"}} +{"category":"resolution/attr_only","id":"attr_only/claude_agent_sdk","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/crewai","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/crewai_task_key","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/crewai_tool_result","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/crewai_flow","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/crewai_flow_node","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/flue","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"flue"}} +{"category":"resolution/attr_only","id":"attr_only/google_adk_prefix","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"google_adk"}} +{"category":"resolution/attr_only","id":"attr_only/google_adk_system","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"google_adk"}} +{"category":"resolution/attr_only","id":"attr_only/haystack","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"haystack"}} +{"category":"resolution/attr_only","id":"attr_only/langchain_prefix","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"2953009462557476175","session_key_hex":"742d31","session_state":6,"vendor":"langchain"}} +{"category":"resolution/attr_only","id":"attr_only/langchain_system","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"langchain"}} +{"category":"resolution/attr_only","id":"attr_only/litellm","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/llamaindex","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"llamaindex"}} +{"category":"resolution/attr_only","id":"attr_only/mastra","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"mastra"}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_provider","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_prefix","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_provider_subclass","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_executor","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_executor_with_message","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_edge_group","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_edge_group_with_message","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"resolution/attr_only","id":"attr_only/microsoft_message_only","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/pydantic_ai","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"pydantic_ai"}} +{"category":"resolution/attr_only","id":"attr_only/pydantic_ai_usage","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"pydantic_ai"}} +{"category":"resolution/attr_only","id":"attr_only/pydantic_ai_logfire_conjunction","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"pydantic_ai"}} +{"category":"resolution/attr_only","id":"attr_only/pydantic_ai_logfire_two_conjuncts_insufficient","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/attr_only","id":"attr_only/semantic_kernel_prefix","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/semantic_kernel_available_functions","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/attr_only","id":"attr_only/semantic_kernel_chat_completions_alone","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/attr_only","id":"attr_only/semantic_kernel_chat_completions_enum_repr","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/attr_only","id":"attr_only/semantic_kernel_chat_completions_semconv_value","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/attr_only","id":"attr_only/semantic_kernel_streaming","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"resolution/attr_only","id":"attr_only/smolagents","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"smolagents"}} +{"category":"resolution/attr_only","id":"attr_only/spring_ai_prefix","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"spring_ai"}} +{"category":"resolution/attr_only","id":"attr_only/spring_ai_system","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"spring_ai"}} +{"category":"resolution/attr_only","id":"attr_only/strands_system","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"strands"}} +{"category":"resolution/attr_only","id":"attr_only/strands_provider","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"strands"}} +{"category":"resolution/attr_only","id":"attr_only/strands_event_loop","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/strands_event_loop_with_operation","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"strands"}} +{"category":"resolution/attr_only","id":"attr_only/vercel_prefix","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:other"}} +{"category":"resolution/attr_only","id":"attr_only/vercel_execute_tool","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"vercel_ai_sdk"}} +{"category":"resolution/attr_only","id":"attr_only/vercel_agent_step","note":"attr-class matcher alone under an unclaimed scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"vercel_ai_sdk"}} +{"category":"resolution/attr_only","id":"attr_only/claude_span_name_with_span_type","note":"scope-rewritten claude dialect: claude_code.* name + span.type classifies","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"claude_agent_sdk"}} +{"category":"resolution/attr_only","id":"attr_only/claude_span_name_without_span_type","note":"the crewai customer-data hazard: claude_code.-named span without span.type","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/effect_ai_guarded_name_without_evidence","note":"an ordinary-English Class.method name with no Effect evidence stays unclaimed","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/effect_ai_strong_name_without_evidence","note":"a module-path name classifies without any Effect evidence (tier 1 unguarded)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"effect_ai"}} +{"category":"resolution/attr_only","id":"attr_only/effect_ai_bare_words_without_evidence","note":"toolChoice + concurrency alone (renamed spans, no Effect evidence) stay unclaimed","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/effect_ai_bare_words_with_evidence","note":"the E3 rename-proof tier: bare-word pair + the @effect/opentelemetry resource","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"effect_ai"}} +{"category":"resolution/attr_only","id":"attr_only/effect_ai_guarded_name_via_scope_is_service","note":"scope.name == service.name is the getTracer idiom, not Effect evidence: unclaimed","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/effect_ai_guarded_name_with_effect_resource","note":"the tier-2 guard that survives: a guarded name under the @effect/opentelemetry resource","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"effect_ai"}} +{"category":"resolution/attr_only","id":"attr_only/spring_ai_openai_without_boot_scope","note":"the openrouter trap: gen_ai.system=openai outside the Boot scope is not spring_ai","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/attr_only","id":"attr_only/spring_ai_openai_under_boot_scope","note":"SP1: a tool-less ChatModel span (no spring.ai.* key) under the Boot scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"spring_ai"}} +{"category":"resolution/attr_only","id":"attr_only/spring_ai_boot_scope_http_span","note":"the reason the Boot scope is conjunct-only: a plain Micrometer HTTP span","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/cross_vendor","id":"cross_vendor/openai_scope_vs_crewai_attr","note":"sufficient scope (band 3xxxx) outranks another vendor's attr hit (2xxxx)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"openinference-openai"}} +{"category":"resolution/cross_vendor","id":"cross_vendor/mastra_resource_vs_agno_attr","note":"sufficient *resource* matcher outranks an attr hit","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"mastra"}} +{"category":"resolution/cross_vendor","id":"cross_vendor/two_attr_bands","note":"two vendors' attr matchers on one span: highest priority wins","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"mastra"}} +{"category":"resolution/cross_vendor","id":"cross_vendor/vendor_beats_unknown","note":"phase 2 flipped this: vercel's ai. evidence is scope-gated, so under an unclaimed scope the unknown tier wins (gen_ai.operation.name → unknown:genai)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/cross_vendor","id":"cross_vendor/unpromoted_candidate_loses_to_unknown","note":"an unpromoted candidate does not suppress the unknown tier","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/cross_vendor","id":"cross_vendor/sufficient_scope_with_foreign_system","note":"sufficient scope wins over a conflicting gen_ai.system attr matcher","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"google_adk"}} +{"category":"resolution/unknown_tier","id":"unknown/genai","note":"present(gen_ai.operation.name)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/unknown_tier","id":"unknown/openinference","note":"present(openinference.span.kind)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:openinference"}} +{"category":"resolution/unknown_tier","id":"unknown/llm_prefix","note":"key_prefix(llm.)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:other"}} +{"category":"resolution/unknown_tier","id":"unknown/traceloop_prefix","note":"key_prefix(traceloop.)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:other"}} +{"category":"resolution/unknown_tier","id":"unknown/ai_prefix","note":"key_prefix(ai.) outside the AI SDK's ai/gen_ai scopes — reachable since phase 2 scope-gated vercel's prefix evidence (fix-queue X3)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:other"}} +{"category":"resolution/unknown_tier","id":"unknown/co_occurrence_gate_off","note":"input.value/output.value alone are deliberately not fingerprints","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/unknown_tier","id":"unknown/genai_in_the_ai_sdk_scope","note":"S11: scope `gen_ai` + gen_ai.operation.name is not vendor evidence — the conjunct is semconv-required, so the clause reduced to \"a tracer named gen_ai claims everything inside it\" over customer data","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/unknown_tier","id":"unknown/co_occurrence_gate_on","note":"the same generic values with an OpenInference attribute present","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:openinference"}} +{"category":"resolution/unknown_tier","id":"unknown/co_occurrence_gate_llm_namespace","note":"generic output.value with the OpenInference llm.* namespace as co-evidence","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:openinference"}} +{"category":"resolution/unknown_tier","id":"unknown/priority_between_buckets","note":"genai outranks openinference outranks llm.","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"resolution/non_ai","id":"non_ai/http_server","note":"ordinary HTTP server span","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/non_ai","id":"non_ai/db_client","note":"ordinary DB client span","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/non_ai","id":"non_ai/no_attributes","note":"no attributes at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/non_ai","id":"non_ai/empty_scope_name","note":"empty scope name","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/non_ai","id":"non_ai/no_scope","note":"ScopeSpans with no InstrumentationScope at all","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"resolution/non_ai","id":"non_ai/no_resource","note":"ResourceSpans with no resource attributes","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"session/agno","id":"session/agno/presence_gated_session_id","note":"no AGENT kind, but session.id present — the phase-2 A3 presence branch makes the session.id candidate authoritative (run.id's stays state 2)","rust":{"rules_version":1,"session_key_hash":"13162659078926236385","session_key_hex":"732d31","session_state":6,"vendor":"agno"}} +{"category":"session/agno","id":"session/agno/state2_not_authoritative","note":"no AGENT kind, no agno.workflow. key, no session.id — both candidates report not-authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"agno"}} +{"category":"session/agno","id":"session/agno/state3_key_absent","note":"authoritative, neither candidate key present","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"agno"}} +{"category":"session/agno","id":"session/agno/state4_empty","note":"authoritative, session.id present but empty; agno.run.id also empty","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"agno"}} +{"category":"session/agno","id":"session/agno/state5_run_only","note":"run-granularity candidate only","rust":{"rules_version":1,"session_key_hash":"16977468257975912505","session_key_hex":"722d39","session_state":5,"vendor":"agno"}} +{"category":"session/agno","id":"session/agno/state6_session","note":"session-granularity candidate resolves; max over candidates","rust":{"rules_version":1,"session_key_hash":"15860208464106379930","session_key_hex":"736573732d61676e6f","session_state":6,"vendor":"agno"}} +{"category":"session/agno","id":"session/agno/state6_via_workflow_prefix","note":"authority via key_prefix(agno.workflow.)","rust":{"rules_version":1,"session_key_hash":"4822631405716103833","session_key_hex":"736573732d61676e6f2d32","session_state":6,"vendor":"agno"}} +{"category":"session/agno","id":"session/agno/state4_empty_wins_over_absent","note":"empty session.id (4) beats absent agno.run.id (3) under max","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"agno"}} +{"category":"session/claude_agent_sdk","id":"session/claude/state2","note":"no span.type ⇒ neither candidate is authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"session/claude_agent_sdk","id":"session/claude/state3","note":"authoritative, no key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"claude_agent_sdk"}} +{"category":"session/claude_agent_sdk","id":"session/claude/state4","note":"present-but-empty session.id","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"claude_agent_sdk"}} +{"category":"session/claude_agent_sdk","id":"session/claude/state5_user_only","note":"user granularity only","rust":{"rules_version":1,"session_key_hash":"1139775732335700012","session_key_hex":"752d37","session_state":5,"vendor":"claude_agent_sdk"}} +{"category":"session/claude_agent_sdk","id":"session/claude/state6","note":"session id present and valid","rust":{"rules_version":1,"session_key_hash":"15639569774906704414","session_key_hex":"736573732d636c61756465","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"session/flue","id":"session/flue/state3_always_candidate_absent","note":"the ALWAYS candidate is authoritative but its key is absent","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"flue"}} +{"category":"session/flue","id":"session/flue/state5_instance","note":"instance granularity resolves at 5","rust":{"rules_version":1,"session_key_hash":"2449292642492405285","session_key_hex":"696e73742d37","session_state":5,"vendor":"flue"}} +{"category":"session/flue","id":"session/flue/state6_conversation","note":"prompt span with a conversation id","rust":{"rules_version":1,"session_key_hash":"12245458656043361916","session_key_hex":"636f6e762d33","session_state":6,"vendor":"flue"}} +{"category":"session/flue","id":"session/flue/state5_decoy_conversation","note":"decoy value 'default' invalidates candidate 1 (4) but instance still resolves (5)","rust":{"rules_version":1,"session_key_hash":"2449292642492405285","session_key_hex":"696e73742d37","session_state":5,"vendor":"flue"}} +{"category":"session/flue","id":"session/flue/state4_decoy_only","note":"decoy conversation id, no instance id ⇒ 4","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"flue"}} +{"category":"session/flue","id":"session/flue/state5_delegate_prompt_rejected","note":"F1: a kind=prompt span carrying flue.task.id is not session-authoritative","rust":{"rules_version":1,"session_key_hash":"2449292642492405285","session_key_hex":"696e73742d37","session_state":5,"vendor":"flue"}} +{"category":"session/google_adk","id":"session/adk/state2","note":"none of the three authority predicates hold","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"google_adk"}} +{"category":"session/google_adk","id":"session/adk/state3","note":"authoritative via gen_ai.system, key absent","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"google_adk"}} +{"category":"session/google_adk","id":"session/adk/state6_via_system","note":"candidate 2 resolves at session granularity","rust":{"rules_version":1,"session_key_hash":"13162659078926236385","session_key_hex":"732d31","session_state":6,"vendor":"google_adk"}} +{"category":"session/google_adk","id":"session/adk/state6_via_conversation","note":"candidate 1's population: invoke_agent","rust":{"rules_version":1,"session_key_hash":"17286432163829527729","session_key_hex":"632d39","session_state":6,"vendor":"google_adk"}} +{"category":"session/google_adk","id":"session/adk/state6_via_invoke_workflow","note":"phase-2 G2: the adk-schema-v2 root (source-cited, zero corpus spans) is authoritative for candidate 1","rust":{"rules_version":1,"session_key_hash":"5030108571139882317","session_key_hex":"632d3130","session_state":6,"vendor":"google_adk"}} +{"category":"session/google_adk","id":"session/adk/state5_invocation_run","note":"run-granularity candidate 3 (authority = present(gen_ai.request.model))","rust":{"rules_version":1,"session_key_hash":"2502602755074883861","session_key_hex":"696e762d31","session_state":5,"vendor":"google_adk"}} +{"category":"session/google_adk","id":"session/adk/state6_max_unions_disjoint","note":"one candidate at 2, one at 6 — max unions instead of cancelling","rust":{"rules_version":1,"session_key_hash":"9028445063357370541","session_key_hex":"632d3131","session_state":6,"vendor":"google_adk"}} +{"category":"session/spring_ai","id":"session/spring/state2","note":"not a chat_client span ⇒ not authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"spring_ai"}} +{"category":"session/spring_ai","id":"session/spring/state3","note":"authoritative, key absent","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"spring_ai"}} +{"category":"session/spring_ai","id":"session/spring/state4_decoy","note":"the 'default' decoy value","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"spring_ai"}} +{"category":"session/spring_ai","id":"session/spring/state4_empty","note":"present-but-empty","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"spring_ai"}} +{"category":"session/spring_ai","id":"session/spring/state6","note":"resolved","rust":{"rules_version":1,"session_key_hash":"17550326829461552516","session_key_hex":"636f6e762d3432","session_state":6,"vendor":"spring_ai"}} +{"category":"session/litellm","id":"session/litellm/state2","note":"no litellm.call_id ⇒ not authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"litellm"}} +{"category":"session/litellm","id":"session/litellm/state3","note":"authoritative, no key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"litellm"}} +{"category":"session/litellm","id":"session/litellm/state4_decoy_default_user","note":"'default_user_id' is a decoy value","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"litellm"}} +{"category":"session/litellm","id":"session/litellm/state4_decoy_empty","note":"the empty string is BOTH a non_empty failure and a declared decoy","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"litellm"}} +{"category":"session/litellm","id":"session/litellm/state5","note":"user granularity resolves at 5, never 6","rust":{"rules_version":1,"session_key_hash":"7407841233302966981","session_key_hex":"757365722d3737","session_state":5,"vendor":"litellm"}} +{"category":"session/langchain","id":"session/langchain/state3","note":"ALWAYS-authoritative candidate with no key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"langchain"}} +{"category":"session/langchain","id":"session/langchain/state4_decoy","note":"'default' decoy","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"langchain"}} +{"category":"session/langchain","id":"session/langchain/state6","note":"resolved thread id","rust":{"rules_version":1,"session_key_hash":"6338205964469458371","session_key_hex":"7468726561642d39","session_state":6,"vendor":"langchain"}} +{"category":"session/mastra","id":"session/mastra/state2","note":"no mastra.span.type ⇒ not authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"mastra"}} +{"category":"session/mastra","id":"session/mastra/state3","note":"authoritative, no candidate key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"mastra"}} +{"category":"session/mastra","id":"session/mastra/state5_run","note":"run granularity","rust":{"rules_version":1,"session_key_hash":"17391958930642725075","session_key_hex":"72756e2d31","session_state":5,"vendor":"mastra"}} +{"category":"session/mastra","id":"session/mastra/state5_user","note":"user granularity","rust":{"rules_version":1,"session_key_hash":"1858641470362393213","session_key_hex":"7265732d31","session_state":5,"vendor":"mastra"}} +{"category":"session/mastra","id":"session/mastra/state6_max","note":"all three candidates resolve; max picks session and the hash follows it","rust":{"rules_version":1,"session_key_hash":"12884826459227698232","session_key_hex":"636f6e762d6d6173747261","session_state":6,"vendor":"mastra"}} +{"category":"session/pydantic_ai","id":"session/pydantic/state2","note":"no gen_ai.operation.name ⇒ not authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"pydantic_ai"}} +{"category":"session/pydantic_ai","id":"session/pydantic/state5_call_id","note":"run granularity only","rust":{"rules_version":1,"session_key_hash":"4226190707766527458","session_key_hex":"63616c6c2d31","session_state":5,"vendor":"pydantic_ai"}} +{"category":"session/pydantic_ai","id":"session/pydantic/state6","note":"conversation id at session granularity","rust":{"rules_version":1,"session_key_hash":"17391958930642725075","session_key_hex":"72756e2d31","session_state":6,"vendor":"pydantic_ai"}} +{"category":"session/pydantic_ai","id":"session/pydantic/state5_uuid7_demotion","note":"phase-2 P1: a strict-UUIDv7 conversation id is pydantic's auto-minted per-run default — value-conditional granularity demotes it to run (5); the hash still comes from the conversation id (candidate-order tie)","rust":{"rules_version":1,"session_key_hash":"7009499996358949202","session_key_hex":"30313839306135642d616339362d373734622d626363652d623330323039396138303537","session_state":5,"vendor":"pydantic_ai"}} +{"category":"session/microsoft_agent_framework","id":"session/maf/state2","note":"operation is not invoke_agent","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"microsoft_agent_framework"}} +{"category":"session/microsoft_agent_framework","id":"session/maf/state4_decoy_local_history","note":"'agent_framework_local_history_persistence' decoy","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"microsoft_agent_framework"}} +{"category":"session/microsoft_agent_framework","id":"session/maf/state4_decoy_unknown","note":"'unknown' decoy","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"microsoft_agent_framework"}} +{"category":"session/microsoft_agent_framework","id":"session/maf/state6","note":"resolved","rust":{"rules_version":1,"session_key_hash":"977815689881094810","session_key_hex":"7468726561645f616263","session_state":6,"vendor":"microsoft_agent_framework"}} +{"category":"session/strands","id":"session/strands/state2","note":"no gen_ai.system / provider ⇒ not authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"strands"}} +{"category":"session/strands","id":"session/strands/state6","note":"authoritative via gen_ai.system","rust":{"rules_version":1,"session_key_hash":"7387820565517287774","session_key_hex":"732d32","session_state":6,"vendor":"strands"}} +{"category":"session/strands","id":"session/strands/state6_via_provider","note":"authoritative via gen_ai.provider.name","rust":{"rules_version":1,"session_key_hash":"2606619711502189936","session_key_hex":"732d33","session_state":6,"vendor":"strands"}} +{"category":"session/openai_agents_sdk","id":"session/openai_agents/state2","note":"no openinference.span.kind ⇒ not authoritative","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"openai_agents_sdk"}} +{"category":"session/openai_agents_sdk","id":"session/openai_agents/state6_session_id","note":"session.id wins; gen_ai.conversation.id ties at 6 and loses on order","rust":{"rules_version":1,"session_key_hash":"5873743417506597526","session_key_hex":"736573732d61","session_state":6,"vendor":"openai_agents_sdk"}} +{"category":"session/openai_agents_sdk","id":"session/openai_agents/state6_conversation_only","note":"second candidate alone","rust":{"rules_version":1,"session_key_hash":"4453951998959152841","session_key_hex":"636f6e762d62","session_state":6,"vendor":"openai_agents_sdk"}} +{"category":"session/smolagents","id":"session/smolagents/state3","note":"authoritative, no key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"smolagents"}} +{"category":"session/smolagents","id":"session/smolagents/state5_user","note":"user granularity only","rust":{"rules_version":1,"session_key_hash":"17164403324138251820","session_key_hex":"752d31","session_state":5,"vendor":"smolagents"}} +{"category":"session/crewai","id":"session/crewai/state3","note":"authority is the scope itself; key absent","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"crewai"}} +{"category":"session/crewai","id":"session/crewai/state6","note":"resolved","rust":{"rules_version":1,"session_key_hash":"8736018979143156140","session_key_hex":"736573732d63726577","session_state":6,"vendor":"crewai"}} +{"category":"session/dspy","id":"session/dspy/state6","note":"scope-gated authority, session granularity","rust":{"rules_version":1,"session_key_hash":"16442119970499362894","session_key_hex":"736573732d64737079","session_state":6,"vendor":"dspy"}} +{"category":"session/llamaindex","id":"session/llamaindex/state3","note":"ALWAYS candidate, key absent","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"llamaindex"}} +{"category":"session/llamaindex","id":"session/llamaindex/state5","note":"run granularity","rust":{"rules_version":1,"session_key_hash":"5399217418148457536","session_key_hex":"72756e2d33","session_state":5,"vendor":"llamaindex"}} +{"category":"session/no_rules","id":"session/no_rules/haystack","note":"vendor with zero session candidates ⇒ state 1","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"haystack"}} +{"category":"session/no_rules","id":"session/no_rules/semantic_kernel","note":"vendor with zero session candidates ⇒ state 1","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"semantic_kernel"}} +{"category":"session/no_rules","id":"session/no_rules/vercel_ai_sdk","note":"vendor with zero session candidates ⇒ state 1","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"vercel_ai_sdk"}} +{"category":"session/no_rules","id":"session/no_rules/unknown_bucket","note":"unknown-tier buckets carry no session rules ⇒ state 1","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"session/no_rules","id":"session/no_rules/effect_ai","note":"effect_ai classifies on span name and has no candidates","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"effect_ai"}} +{"category":"values/typed","id":"typed/bool_true_resource","note":"BoolValue(true) canonicalizes to 'true' for eq()","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"langchain"}} +{"category":"values/typed","id":"typed/bool_false_resource","note":"BoolValue(false) must NOT satisfy eq(..., 'true')","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"values/typed","id":"typed/session_key_int","note":"IntValue beyond 2^32 as a session key","rust":{"rules_version":1,"session_key_hash":"3314592284979140625","session_key_hex":"34323934393637323936","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_int_negative","note":"IntValue::MIN as a session key","rust":{"rules_version":1,"session_key_hash":"15897920141151212888","session_key_hex":"2d39323233333732303336383534373735383038","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_double","note":"DoubleValue canonicalizes with Rust's float formatting","rust":{"rules_version":1,"session_key_hash":"7942122020128294879","session_key_hex":"312e35","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_double_integral","note":"42.0 renders as '42', not '42.0'","rust":{"rules_version":1,"session_key_hash":"985280342255011280","session_key_hex":"3432","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_bool","note":"BoolValue as a session key","rust":{"rules_version":1,"session_key_hash":"5512790967622792941","session_key_hex":"74727565","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_bytes","note":"BytesValue hex-encodes (the row writer's rule, not JSON.stringify)","rust":{"rules_version":1,"session_key_hash":"1429165956392325225","session_key_hex":"646561646265656630303031","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_array","note":"ArrayValue renders as a JSON array of canonical strings","rust":{"rules_version":1,"session_key_hash":"12678579444740363438","session_key_hex":"5b2261222c2232222c2266616c7365225d","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_kvlist","note":"KvlistValue renders as a JSON object; the row Map's key order is what SQL sees","rust":{"rules_version":1,"session_key_hash":"5864374712609636384","session_key_hex":"7b2261223a226669727374222c226d223a2237222c227a223a226c617374227d","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_nested","note":"nested array/kvlist","rust":{"rules_version":1,"session_key_hash":"1070448931733804658","session_key_hex":"5b227b5c226b5c223a5c22765c227d222c225b5c22315c225d225d","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_untyped","note":"AnyValue with no value ⇒ empty string ⇒ state 4","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/session_key_valueless","note":"KeyValue with no AnyValue ⇒ empty string ⇒ state 4","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"claude_agent_sdk"}} +{"category":"values/typed","id":"typed/eq_int_vs_string","note":"IntValue on gen_ai.system cannot match any vendor's string literal","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"values/typed","id":"typed/eq_kvlist_on_matched_key","note":"KvlistValue on an eq()-compared key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"values/typed","id":"typed/present_only_key_is_type_blind","note":"present() ignores the value's type entirely","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"values/typed","id":"typed/array_on_prefix_key","note":"key_prefix() ignores values; the key alone decides","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:other"}} +{"category":"values/present_empty","id":"present_empty/unknown_genai","note":"gen_ai.operation.name = '' still fingerprints as unknown:genai","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"values/present_empty","id":"present_empty/unknown_openinference","note":"openinference.span.kind = ''","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:openinference"}} +{"category":"values/present_empty","id":"present_empty/eq_matcher_not_satisfied","note":"an empty value cannot satisfy eq() against a non-empty literal","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"values/present_empty","id":"present_empty/prefix_key_empty_value","note":"key_prefix() reads keys, so an empty value still hits","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"spring_ai"}} +{"category":"values/present_empty","id":"present_empty/empty_key_name","note":"an attribute whose key is the empty string","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"values/present_empty","id":"present_empty/state4_vs_state3_empty","note":"present-but-empty ⇒ 4","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"claude_agent_sdk"}} +{"category":"values/present_empty","id":"present_empty/state4_vs_state3_absent","note":"the same span with the key absent ⇒ 3","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"claude_agent_sdk"}} +{"category":"values/present_empty","id":"present_empty/authority_key_empty","note":"the authority predicate is present(), so an empty span.type still grants it","rust":{"rules_version":1,"session_key_hash":"18315090338952981153","session_key_hex":"736573732d656d7074792d61757468","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"values/present_empty","id":"present_empty/eq_authority_empty","note":"eq()-based authority against an empty value","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"flue"}} +{"category":"keys/duplicate","id":"duplicate/gen_ai_system_spring_first","note":"duplicate registry key: the first occurrence decides both the verdict and the row","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"spring_ai"}} +{"category":"keys/duplicate","id":"duplicate/gen_ai_system_strands_first","note":"duplicate registry key: the first occurrence decides both the verdict and the row","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"strands"}} +{"category":"keys/duplicate","id":"duplicate/session_id_valid_then_empty","note":"first occurrence valid, second empty ⇒ state 6","rust":{"rules_version":1,"session_key_hash":"13732450810557198517","session_key_hex":"736573732d6475702d31","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"keys/duplicate","id":"duplicate/session_id_empty_then_valid","note":"first occurrence empty, second valid ⇒ state 4","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":4,"vendor":"claude_agent_sdk"}} +{"category":"keys/duplicate","id":"duplicate/session_id_typed_then_string","note":"typed first occurrence wins over a later string","rust":{"rules_version":1,"session_key_hash":"5566429635965498611","session_key_hex":"37","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"keys/duplicate","id":"duplicate/authority_key_duplicated","note":"the authority key duplicated with a contradicting second value","rust":{"rules_version":1,"session_key_hash":"8957691506115056961","session_key_hex":"736573732d6475702d33","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"keys/duplicate","id":"duplicate/authority_key_contradiction_first","note":"the mirror image: the non-matching value comes first","rust":{"rules_version":1,"session_key_hash":"2162908562136087333","session_key_hex":"736573732d6475702d34","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"keys/duplicate","id":"duplicate/triplicate","note":"three occurrences of a registry key","rust":{"rules_version":1,"session_key_hash":"15096619121631409600","session_key_hex":"6669727374","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"keys/duplicate","id":"duplicate/non_registry_key_last_wins","note":"a key no rule consults keeps the LAST occurrence in the row Map","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"keys/duplicate","id":"duplicate/prefix_family_duplicated","note":"duplicated keys inside a key_prefix family","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"spring_ai"}} +{"category":"keys/duplicate","id":"duplicate/resource_and_scope_same_key","note":"the same registry key on the resource, the scope and the span","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"spring_ai"}} +{"category":"keys/duplicate","id":"duplicate/scope_attribute_duplicated","note":"duplicate keys inside the scope attribute list","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/gen_ai.systen","note":"same length, last byte differs","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/gen_ai.systemm","note":"one byte longer","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/gen_ai.syste","note":"one byte shorter","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/Gen_ai.system","note":"case-flipped first byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/gen_ai.operation.namf","note":"same length as a fingerprint key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/session.iD","note":"case-flipped tail","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/sessionXid","note":"separator replaced","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/span.typ","note":"prefix of a registry key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/span.typee","note":"registry key plus a byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/spring.ai","note":"the prefix without its trailing dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/spring/ai.kind","note":"separator swapped inside a prefix","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/spring.ao.kind","note":"one byte inside the prefix differs","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/agno","note":"prefix minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/agnos.run","note":"prefix plus a byte before the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/ai","note":"the shortest prefix minus its dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/aix.thing","note":"shares the first byte of the ai. prefix","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/llm","note":"llm. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/llmx.model","note":"llm prefix near miss","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/traceloop","note":"traceloop. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/traceloo.x","note":"one byte short inside the prefix","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/flue","note":"flue. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/mastra","note":"mastra. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/model_i","note":"resource key minus a byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/model_idx","note":"resource key plus a byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/telemetry.sdk.nam","note":"sufficient-resource key minus a byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/openinference.span.kin","note":"fingerprint key minus a byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/crew","note":"crew_ prefix minus the underscore","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/flow","note":"flow_ prefix minus the underscore","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/event_loop","note":"event_loop. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/executor","note":"executor. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/edge_group","note":"edge_group. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/sk","note":"sk. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/smolagents","note":"smolagents. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/langsmith","note":"langsmith. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/litellm","note":"litellm. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/haystack","note":"haystack. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/llamaindex","note":"llamaindex. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/pydantic_ai","note":"pydantic_ai. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/agent_framework","note":"agent_framework. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/gcp.vertex.agent","note":"gcp.vertex.agent. minus the dot","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/openinference.instrumentation.agn","note":"one byte short","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/openinference.instrumentation.agnoo","note":"one byte long","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/Openinference.instrumentation.agno","note":"case-flipped","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/openinference.instrumentation.agno ","note":"trailing space","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/ openinference.instrumentation.agno","note":"leading space","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/pydantic_ai","note":"underscore instead of the hyphen","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/langsmit","note":"one byte short","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"keys/near_miss","id":"near_miss/scope/litellmx","note":"one byte long","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"unicode","id":"unicode/session_value/emoji","note":"astral-plane characters","rust":{"rules_version":1,"session_key_hash":"11038541917872144123","session_key_hex":"f09f9982f09fa7a0f09f9a80","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/cjk","note":"3-byte sequences","rust":{"rules_version":1,"session_key_hash":"15238778173770232695","session_key_hex":"e4bc9ae8a9b12de8ad98e588a5e5ad902d3432","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/rtl","note":"RTL text","rust":{"rules_version":1,"session_key_hash":"8593480859355344406","session_key_hex":"d79ed796d794d7942dd7a9d799d797d794","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/combining","note":"combining marks after ASCII","rust":{"rules_version":1,"session_key_hash":"10252815871106751813","session_key_hex":"65cc81cca773657373696f6e","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/zero_width","note":"zero-width joiners inside the value","rust":{"rules_version":1,"session_key_hash":"5413262754819770266","session_key_hex":"73657373e2808be2808d696f6e","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/bom","note":"a leading byte-order mark","rust":{"rules_version":1,"session_key_hash":"9521285589434588519","session_key_hex":"efbbbf73657373696f6e2d31","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/surrogate_pair_boundary","note":"the last valid scalar value","rust":{"rules_version":1,"session_key_hash":"9018560289838767314","session_key_hex":"616161f48fbfbf","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/nul_adjacent","note":"an embedded NUL","rust":{"rules_version":1,"session_key_hash":"474006516132503015","session_key_hex":"6265666f7265006166746572","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/nul_leading","note":"a leading NUL","rust":{"rules_version":1,"session_key_hash":"2896917956751875279","session_key_hex":"006c656164696e67","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/nul_trailing","note":"a trailing NUL","rust":{"rules_version":1,"session_key_hash":"2730887940541437767","session_key_hex":"747261696c696e6700","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/control_chars","note":"C0 control characters","rust":{"rules_version":1,"session_key_hash":"15664684759887450300","session_key_hex":"6101021f62","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/quotes_and_backslash","note":"SQL-hostile punctuation in a hashed value","rust":{"rules_version":1,"session_key_hash":"595630659587464574","session_key_hex":"697427732061205c20227465737422202d2d202f2a202a2f","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/session_value/newlines","note":"newlines and tabs (NDJSON is line-delimited)","rust":{"rules_version":1,"session_key_hash":"8866067739223767042","session_key_hex":"6c696e65310a6c696e65320d0a6c696e653309","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"unicode","id":"unicode/key/🌍.emoji.key","note":"a 4-byte lead byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"unicode","id":"unicode/key/gen_ai.系统","note":"multi-byte tail on a registry-ish key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"unicode","id":"unicode/key/ai.🙂","note":"multi-byte tail inside a registry prefix","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:other"}} +{"category":"unicode","id":"unicode/key/аi.operationId","note":"Cyrillic 'а' homoglyph as the first byte","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"unicode","id":"unicode/key/gen_ai.system\u0000","note":"a trailing NUL in the key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"unicode","id":"unicode/key/gen_ai.system","note":"a BOM in front of a registry key","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"unicode","id":"unicode/scope_name","note":"a multi-byte near miss on a sufficient scope matcher","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:openinference"}} +{"category":"unicode","id":"unicode/span_name","note":"a span name that is a unicode near miss of an effect_ai matcher","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"unicode","id":"unicode/session_value","note":"a non-ASCII session-key value","rust":{"rules_version":1,"session_key_hash":"18045091784994185365","session_key_hex":"e382bbe38383e382b7e383a7e383b32d31","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"oversized","id":"oversized/64kib_session_value","note":"a 64 KiB session key value, hashed in full","rust":{"rules_version":1,"session_key_hash":"7524359276279220832","session_key_hex":"78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"oversized","id":"oversized/16kib_unicode_session_value","note":"16 Ki astral characters (64 KiB of UTF-8)","rust":{"rules_version":1,"session_key_hash":"12360671198098928768","session_key_hex":"f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982f09f9982","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"oversized","id":"oversized/wide_attribute_list","note":"60 non-registry attributes around the two that matter","rust":{"rules_version":1,"session_key_hash":"4391406247644661733","session_key_hex":"736573732d77696465","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"oversized","id":"oversized/spilled_registry_keys","note":"more registry keys than the inline attribute view holds","rust":{"rules_version":1,"session_key_hash":"6419705041821120113","session_key_hex":"736573732d7370696c6c","session_state":6,"vendor":"claude_agent_sdk"}} +{"category":"oversized","id":"oversized/long_key","note":"a 4 KiB key inside a registry prefix family","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"spring_ai"}} +{"category":"oversized","id":"oversized/deep_array","note":"a deeply nested array value","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:other"}} +{"category":"oversized","id":"oversized/long_span_name","note":"a 4 KiB span name","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"pseudo_keys","id":"pseudo/span_attribute_named_scope_name","note":"a span attribute literally called scope.name must not shadow the column","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"agno"}} +{"category":"pseudo_keys","id":"pseudo/span_attribute_named_span_name","note":"a span attribute called span.name against effect_ai's span-name matchers","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"pseudo_keys","id":"pseudo/effect_ai_span_name_match","note":"the real span-name matcher, promoted by the resource candidate","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"effect_ai"}} +{"category":"pseudo_keys","id":"pseudo/scope_version_and_schema_url","note":"scope.version / scope.schema_url carry values (no matcher reads them today)","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:genai"}} +{"category":"pseudo_keys","id":"pseudo/empty_scope_version","note":"empty scope version and schema url","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"unknown:openinference"}} +{"category":"pseudo_keys","id":"pseudo/scope_name_eq_is_case_sensitive","note":"vercel's insufficient scope matcher, exact match","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":1,"vendor":"vercel_ai_sdk"}} +{"category":"cross_class","id":"cross_class/resource_key_on_span","note":"a resource-class matcher's key carried as a span attribute","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"cross_class","id":"cross_class/resource_key_shadowed_by_span","note":"the resource carries the matching value; a span attribute of the same name disagrees","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"mastra"}} +{"category":"cross_class","id":"cross_class/attr_prefix_on_resource","note":"an attr-class key_prefix family carried on the resource","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"cross_class","id":"cross_class/attr_key_on_scope","note":"an attr-class eq() key carried on the scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"cross_class","id":"cross_class/session_key_on_resource","note":"the session-candidate key carried on the resource instead of the span","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"claude_agent_sdk"}} +{"category":"cross_class","id":"cross_class/session_key_on_scope","note":"the session-candidate key carried on the scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":3,"vendor":"claude_agent_sdk"}} +{"category":"cross_class","id":"cross_class/authority_key_on_resource","note":"the authority predicate's key carried on the resource","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":2,"vendor":"claude_agent_sdk"}} +{"category":"cross_class","id":"cross_class/unknown_fingerprint_on_resource","note":"an unknown-tier fingerprint key carried on the resource","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"cross_class","id":"cross_class/unknown_fingerprint_on_scope","note":"an unknown-tier fingerprint key carried on the scope","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} +{"category":"cross_class","id":"cross_class/scope_class_key_on_span","note":"a scope-class matcher keyed on a real attribute, carried on the span","rust":{"rules_version":1,"session_key_hash":"0","session_key_hex":null,"session_state":0,"vendor":""}} diff --git a/apps/ingest/src/ai_adversarial_fixtures.rs b/apps/ingest/src/ai_adversarial_fixtures.rs new file mode 100644 index 000000000..a9ccfe85b --- /dev/null +++ b/apps/ingest/src/ai_adversarial_fixtures.rs @@ -0,0 +1,3033 @@ +//! Adversarial classification fixtures: hand-built hostile spans, driven through +//! the **real** row writer. +//! +//! # What this is +//! +//! `fixtures/classification/` is real wire data — what the vendors actually emit. +//! This module is the opposite corner: a deterministic corpus of spans nobody +//! sent, constructed to sit on the classifier's edges. Typed and valueless +//! `AnyValue`s, present-but-empty values, duplicate keys, near-miss key spellings, +//! astral-plane and NUL-bearing UTF-8, 64 KiB values that spill out of the inline +//! attribute path, one span carrying six vendors' evidence at once, and the full +//! session-state ladder per vendor. +//! +//! Every case runs through [`super::encode_traces`] — the same function the ingest +//! request path calls, with the classification flag on — so the fixture is a golden +//! for the classifier **as the write path invokes it**, not for the classifier +//! called in isolation. The generator additionally asserts, per span, that the +//! `ai_vendor` / `ai_session_key_state` / `ai_session_key_hash` / `ai_rules_version` +//! the row writer emitted equal what a direct [`ResourceContext`] call produces, and +//! that the hash equals `cityhash102::city_hash64` over the winning key — so the row +//! writer and the classifier cannot drift apart silently. +//! +//! The checked-in artifact (`fixtures/adversarial/adversarial-spans.jsonl`) carries +//! the verdict per case, plus the hex of the raw winning session-key value, which is +//! never stored on a row. That hex is what +//! `packages/domain/src/ai/hash-alignment.clickhouse.e2e.test.ts` feeds to a real +//! ClickHouse to prove `cityHash64` there equals `city_hash64` here — the hash +//! contract the read path's "recompute the column in SQL" claim rests on. +//! +//! # What this does not prove +//! +//! Nothing about the trace-capture reference evaluator (`scripts/verify-seed.ts`), +//! which is a third implementation: it renders kvlist values in insertion order and +//! `JSON.stringify`s bytes, where `any_value_string` sorts kvlist keys (the row Map +//! is a `serde_json::Map`) and hex-encodes bytes. "The fixture agrees with the seed +//! verifier" is not a claim made here. +//! +//! # Determinism +//! +//! No clock, no RNG, no environment. Receive time, span start times, trace/span ids +//! and every attribute value are constants or derived from a fixed counter, so +//! regenerating the artifact is byte-stable; [`fixture_is_reproducible`] pins that, +//! which is what turns it into a golden: any rule change that moves a verdict fails +//! `cargo test` with the moved line. +//! +//! # Regeneration +//! +//! ```sh +//! ADVERSARIAL_FIXTURE_OUT=apps/ingest/fixtures/adversarial/adversarial-spans.jsonl \ +//! cargo test -p maple-ingest --lib write_adversarial_fixture -- --ignored --nocapture +//! ``` + +use std::collections::BTreeSet; + +use opentelemetry_proto::tonic::common::v1::{ArrayValue, InstrumentationScope, KeyValueList}; +use opentelemetry_proto::tonic::resource::v1::Resource; +use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans}; + +use super::*; +use crate::ai_classifier::ResourceContext; + +// --------------------------------------------------------------------------- +// constants +// --------------------------------------------------------------------------- + +/// Batch receive time, epoch seconds — 2023-11-14 22:13:20 UTC. Fixed so +/// `ai_rollup_hour` never depends on when the fixture was generated. +const RECEIVE_SECS: i64 = 1_700_000_000; +/// Every span starts at the receive second, inside the rollup clamp window. +const START_NANOS: u64 = RECEIVE_SECS as u64 * 1_000_000_000; + +const ORG_ID: &str = "org_adversarial"; + +// --------------------------------------------------------------------------- +// attribute helpers +// --------------------------------------------------------------------------- + +fn any(value: any_value::Value) -> Option { + Some(AnyValue { value: Some(value) }) +} + +/// String-valued attribute. +fn s(key: &str, value: &str) -> KeyValue { + KeyValue { + key: key.to_string(), + value: any(any_value::Value::StringValue(value.to_string())), + } +} + +fn int(key: &str, value: i64) -> KeyValue { + KeyValue { + key: key.to_string(), + value: any(any_value::Value::IntValue(value)), + } +} + +fn double(key: &str, value: f64) -> KeyValue { + KeyValue { + key: key.to_string(), + value: any(any_value::Value::DoubleValue(value)), + } +} + +fn boolean(key: &str, value: bool) -> KeyValue { + KeyValue { + key: key.to_string(), + value: any(any_value::Value::BoolValue(value)), + } +} + +fn bytes(key: &str, value: &[u8]) -> KeyValue { + KeyValue { + key: key.to_string(), + value: any(any_value::Value::BytesValue(value.to_vec())), + } +} + +fn array(key: &str, values: Vec) -> KeyValue { + KeyValue { + key: key.to_string(), + value: any(any_value::Value::ArrayValue(ArrayValue { + values: values.into_iter().filter_map(|kv| kv.value).collect(), + })), + } +} + +fn kvlist(key: &str, values: Vec) -> KeyValue { + KeyValue { + key: key.to_string(), + value: any(any_value::Value::KvlistValue(KeyValueList { values })), + } +} + +/// `AnyValue { value: None }` — a wire-legal "typed nothing". +fn untyped(key: &str) -> KeyValue { + KeyValue { + key: key.to_string(), + value: Some(AnyValue { value: None }), + } +} + +/// `KeyValue { value: None }` — the attribute exists, the value field does not. +fn valueless(key: &str) -> KeyValue { + KeyValue { + key: key.to_string(), + value: None, + } +} + +fn scope(name: &str) -> InstrumentationScope { + InstrumentationScope { + name: name.to_string(), + version: "1.2.3".to_string(), + attributes: Vec::new(), + dropped_attributes_count: 0, + } +} + +// --------------------------------------------------------------------------- +// spec model +// --------------------------------------------------------------------------- + +/// One span to classify. +struct Case { + id: String, + category: &'static str, + note: String, + span_name: String, + attributes: Vec, +} + +fn case( + category: &'static str, + id: &str, + note: &str, + span_name: &str, + attributes: Vec, +) -> Case { + Case { + id: id.to_string(), + category, + note: note.to_string(), + span_name: span_name.to_string(), + attributes, + } +} + +/// A `ResourceSpans`/`ScopeSpans` pair and the spans under it. Groups with more than +/// one case exercise the hoisting path — resource and scope evidence resolved once and +/// shared — which a corpus of singletons would never reach. +struct Group { + resource: Vec, + scope: Option, + schema_url: String, + cases: Vec, +} + +impl Group { + fn new(scope_name: &str, cases: Vec) -> Self { + Self { + resource: vec![s("service.name", "adversarial-fixture")], + scope: Some(scope(scope_name)), + schema_url: String::new(), + cases, + } + } + + fn resource(mut self, attributes: Vec) -> Self { + self.resource = attributes; + self + } + + fn scope_attrs(mut self, attributes: Vec) -> Self { + if let Some(scope) = self.scope.as_mut() { + scope.attributes = attributes; + } + self + } + + fn scope_version(mut self, version: &str) -> Self { + if let Some(scope) = self.scope.as_mut() { + scope.version = version.to_string(); + } + self + } + + fn no_scope(mut self) -> Self { + self.scope = None; + self + } + + fn schema_url(mut self, url: &str) -> Self { + self.schema_url = url.to_string(); + self + } + +} + +/// A single-span group — the common shape. +fn one(scope_name: &str, case: Case) -> Group { + Group::new(scope_name, vec![case]) +} + +// --------------------------------------------------------------------------- +// the corpus +// --------------------------------------------------------------------------- + +/// Neutral scope for spans whose classification must come from attributes alone. +const NEUTRAL_SCOPE: &str = "com.example.app"; + +fn groups() -> Vec { + let mut out = Vec::new(); + out.extend(sufficient_scope_groups()); + out.extend(resource_and_promotion_groups()); + out.extend(attr_only_groups()); + out.extend(cross_vendor_groups()); + out.extend(unknown_tier_groups()); + out.extend(non_ai_groups()); + out.extend(session_state_groups()); + out.extend(typed_value_groups()); + out.extend(present_but_empty_groups()); + out.extend(duplicate_key_groups()); + out.extend(near_miss_groups()); + out.extend(unicode_groups()); + out.extend(oversized_groups()); + out.extend(pseudo_key_groups()); + out.extend(cross_class_groups()); + out +} + +/// Every vendor whose scope matcher is `sufficient` — the branch where resource/scope +/// evidence classifies on its own, with no span attribute involved. +fn sufficient_scope_groups() -> Vec { + const SCOPES: &[(&str, &str)] = &[ + ("openinference.instrumentation.agno", "agno"), + ("com.anthropic.claude_code.tracing", "claude_agent_sdk"), + ("com.anthropic.claude_code.events", "claude_agent_sdk"), + ("com.anthropic.claude_code", "claude_agent_sdk"), + ("openinference.instrumentation.crewai", "crewai"), + ("crewai.telemetry", "crewai"), + ("openinference.instrumentation.dspy", "dspy"), + ("@flue/opentelemetry", "flue"), + ("gcp.vertex.agent", "google_adk"), + ("haystack", "haystack"), + ("langsmith", "langchain"), + ("litellm", "litellm"), + ("llamaindex.opentelemetry.tracer", "llamaindex"), + ("@mastra/otel-exporter", "mastra"), + ("agent_framework", "microsoft_agent_framework"), + ( + "openinference.instrumentation.openai_agents", + "openai_agents_sdk", + ), + ( + "openinference.instrumentation.openai", + "openinference-openai", + ), + ("pydantic-ai", "pydantic_ai"), + ( + "semantic_kernel.utils.telemetry.agent_diagnostics.decorators", + "semantic_kernel", + ), + ( + "semantic_kernel.utils.telemetry.model_diagnostics.decorators", + "semantic_kernel", + ), + ( + "semantic_kernel.functions.kernel_function", + "semantic_kernel", + ), + ( + "semantic_kernel.connectors.ai.chat_completion_client_base", + "semantic_kernel", + ), + ("agent_runtime InProcessRuntime", "semantic_kernel"), + ("openinference.instrumentation.smolagents", "smolagents"), + ("strands.telemetry.tracer", "strands"), + // Phase-2 SK1/SK2: the module-path enumeration and the pinned + // `agent_runtime InProcessRuntime` value became prefix families, so a + // utils/telemetry refactor and a non-InProcess CoreRuntime still classify. + ( + "semantic_kernel.utils.telemetry.some_future_module", + "semantic_kernel", + ), + ("agent_runtime SomeOtherRuntime", "semantic_kernel"), + ]; + SCOPES + .iter() + .enumerate() + .map(|(index, (scope_name, vendor))| { + Group::new( + scope_name, + vec![ + case( + "resolution/sufficient_scope", + &format!("sufficient_scope/{vendor}/{index}"), + &format!("sufficient scope matcher for {vendor}, no AI attribute at all"), + "operation", + vec![s("http.route", "/x")], + ), + case( + "resolution/sufficient_scope", + &format!("sufficient_scope_hoisted/{vendor}/{index}"), + "second span under the same hoisted scope", + "operation.two", + vec![], + ), + ], + ) + }) + .collect() +} + +/// Insufficient resource/scope matchers: promoted by a same-vendor attr hit, and the +/// same evidence without one (the negative the plan names explicitly). +fn resource_and_promotion_groups() -> Vec { + // (label, resource attrs, scope name, promoting attrs) + let cases: Vec<(&str, Vec, &str, Vec)> = vec![ + ( + "spring_ai_scope", + vec![s("service.name", "spring-app")], + "org.springframework.boot", + vec![s("spring.ai.kind", "chat_client")], + ), + ( + "vercel_scope_ai", + vec![s("service.name", "next-app")], + "ai", + vec![s("ai.operationId", "ai.generateText")], + ), + ( + "vercel_scope_gen_ai", + vec![s("service.name", "next-app")], + "gen_ai", + vec![s("gen_ai.operation.name", "agent_step")], + ), + ( + "claude_resource", + vec![s("service.name", "claude-code")], + NEUTRAL_SCOPE, + vec![s("span.type", "interaction")], + ), + ( + "litellm_resource", + vec![s("service.name", "gateway"), s("model_id", "gpt-4o-mini")], + NEUTRAL_SCOPE, + vec![s("litellm.call_id", "call-1")], + ), + ( + "llamaindex_resource", + vec![s("service.name", "llamaindex.opentelemetry")], + NEUTRAL_SCOPE, + vec![s("llamaindex.run_id", "run-1")], + ), + ( + "langchain_resource", + vec![ + s("service.name", "langgraph-app"), + s("langsmith.internal_provider", "true"), + ], + NEUTRAL_SCOPE, + vec![s("langsmith.trace.name", "chain")], + ), + ( + "effect_ai_resource", + vec![ + s("service.name", "effect-app"), + s("telemetry.sdk.name", "@effect/opentelemetry"), + ], + NEUTRAL_SCOPE, + vec![], + ), + ]; + + let mut out = Vec::new(); + for (label, resource, scope_name, promoting) in cases { + // effect_ai promotes on span *name*, not an attribute. + let span_name = if label == "effect_ai_resource" { + "LanguageModel.generateText" + } else { + "op" + }; + let mut promoted = promoting.clone(); + promoted.push(s("http.route", "/api")); + out.push( + Group::new( + scope_name, + vec![case( + "resolution/insufficient_promoted", + &format!("promoted/{label}"), + "insufficient resource/scope candidate promoted by a same-vendor hit", + span_name, + promoted, + )], + ) + .resource(resource.clone()), + ); + out.push( + Group::new( + scope_name, + vec![case( + "resolution/insufficient_not_promoted", + &format!("not_promoted/{label}"), + "same insufficient evidence with nothing to promote it", + "POST", + vec![s("http.request.method", "POST"), s("http.route", "/api")], + )], + ) + .resource(resource), + ); + } + out +} + +/// Attr-class matchers on their own, under a scope no rule claims. +fn attr_only_groups() -> Vec { + let attrs: Vec<(&str, Vec)> = vec![ + ("agno", vec![s("agno.run.id", "r-1")]), + ("claude_agent_sdk", vec![s("span.type", "tool.execution")]), + ("crewai", vec![s("crew_key", "k")]), + ("crewai_task_key", vec![s("task_key", "research")]), + ( + "crewai_tool_result", + vec![s("tool.result_as_answer", "false")], + ), + ("crewai_flow", vec![s("flow_name", "f")]), + ("crewai_flow_node", vec![s("flow.node.id", "n")]), + ("flue", vec![s("flue.operation.kind", "tool")]), + ( + "google_adk_prefix", + vec![s("gcp.vertex.agent.invocation_id", "i-1")], + ), + ( + "google_adk_system", + vec![s("gen_ai.system", "gcp.vertex.agent")], + ), + ("haystack", vec![s("haystack.component.name", "retriever")]), + ( + "langchain_prefix", + vec![s("langsmith.metadata.thread_id", "t-1")], + ), + ("langchain_system", vec![s("gen_ai.system", "langchain")]), + ("litellm", vec![s("litellm.call_id", "c-1")]), + ("llamaindex", vec![s("llamaindex.span.kind", "query")]), + ("mastra", vec![s("mastra.span.type", "agent_run")]), + ( + "microsoft_provider", + vec![s("gen_ai.provider.name", "microsoft.agent_framework")], + ), + ("microsoft_prefix", vec![s("agent_framework.run.id", "r")]), + // Phase-2 MS1: the provider fingerprint is a value PREFIX, so the harness + // subclass's `microsoft.agent_framework.harness` is covered too. + ( + "microsoft_provider_subclass", + vec![s("gen_ai.provider.name", "microsoft.agent_framework.harness")], + ), + // Phase-2 X4 sweep: `executor.` / `edge_group.` are generic scheduler + // vocabulary (the class that already burned `workflow.` on 2,037 eve_slack + // spans), so they classify only alongside the co-emitted `message.*` keys. + ("microsoft_executor", vec![s("executor.id", "e")]), + ( + "microsoft_executor_with_message", + vec![ + s("executor.id", "e"), + s("executor.type", "AgentExecutor"), + s("message.type", "ConcurrentRequestMessage"), + ], + ), + ( + "microsoft_edge_group", + vec![s("edge_group.type", "fan_out")], + ), + ( + "microsoft_edge_group_with_message", + vec![ + s("edge_group.type", "fan_out"), + s("message.source_id", "dispatcher"), + ], + ), + ( + "microsoft_message_only", + vec![s("message.type", "ConcurrentResponseMessage")], + ), + ("pydantic_ai", vec![s("pydantic_ai.all_messages", "[]")]), + ( + "pydantic_ai_usage", + vec![s("gen_ai.aggregated_usage.input_tokens", "12")], + ), + // Phase-2 P2: the hardened logfire fallback tier — the 3-way conjunction + // fires; the queue's 2-conjunct form must NOT (logfire.* is a cross-vendor + // dialect; without pydantic-unique co-evidence the span stays unknown:genai). + ( + "pydantic_ai_logfire_conjunction", + vec![ + s("logfire.json_schema", "{\"type\":\"object\"}"), + s("gen_ai.operation.name", "chat"), + s("operation.cost", "0.00013"), + ], + ), + ( + "pydantic_ai_logfire_two_conjuncts_insufficient", + vec![ + s("logfire.json_schema", "{\"type\":\"object\"}"), + s("gen_ai.operation.name", "chat"), + ], + ), + // Phase-2 X4: the two-letter `sk.` prefix is gone — only the exact + // `sk.available_functions` key classifies on scope loss, so a generic + // `sk.*` app key (Sidekiq, an `sdk` typo) no longer claims the vendor. + ("semantic_kernel_prefix", vec![s("sk.function.name", "f")]), + ( + "semantic_kernel_available_functions", + vec![s("sk.available_functions", "[]")], + ), + // Phase-2 clause 5: the non-streaming chat fallback is conjunctive on SK's + // Python-enum-repr finish_reason; the operation value alone must not fire. + ( + "semantic_kernel_chat_completions_alone", + vec![s("gen_ai.operation.name", "chat.completions")], + ), + ( + "semantic_kernel_chat_completions_enum_repr", + vec![ + s("gen_ai.operation.name", "chat.completions"), + s("gen_ai.response.finish_reason", "FinishReason.TOOL_CALLS"), + ], + ), + ( + "semantic_kernel_chat_completions_semconv_value", + vec![ + s("gen_ai.operation.name", "chat.completions"), + s("gen_ai.response.finish_reason", "tool_calls"), + ], + ), + ( + "semantic_kernel_streaming", + vec![s("gen_ai.operation.name", "chat.streaming_completions")], + ), + ("smolagents", vec![s("smolagents.max_steps", "10")]), + ("spring_ai_prefix", vec![s("spring.ai.kind", "chat_client")]), + ("spring_ai_system", vec![s("gen_ai.system", "spring_ai")]), + ("strands_system", vec![s("gen_ai.system", "strands-agents")]), + ( + "strands_provider", + vec![s("gen_ai.provider.name", "strands-agents")], + ), + // Phase-2 X4 sweep: `event_loop.` is generic async-runtime vocabulary, not a + // vendor namespace, so it needs generic-AI co-evidence (a vendor-owned guard + // would collapse the clause into the gen_ai.system one). All 19 corpus carriers + // have gen_ai.operation.name; an asyncio monitor has none. + ("strands_event_loop", vec![s("event_loop.cycle_id", "c")]), + ( + "strands_event_loop_with_operation", + vec![ + s("event_loop.cycle_id", "c"), + s("gen_ai.operation.name", "execute_event_loop_cycle"), + ], + ), + ( + "vercel_prefix", + vec![s("ai.operationId", "ai.generateText")], + ), + ( + "vercel_execute_tool", + vec![s("gen_ai.execute_tool.duration", "1")], + ), + ( + "vercel_agent_step", + vec![s("gen_ai.operation.name", "agent_step")], + ), + ]; + let mut out: Vec = attrs + .into_iter() + .map(|(label, attributes)| { + one( + NEUTRAL_SCOPE, + case( + "resolution/attr_only", + &format!("attr_only/{label}"), + "attr-class matcher alone under an unclaimed scope", + "op", + attributes, + ), + ) + }) + .collect(); + // Phase-2 C2 witnesses for claude_agent_sdk's scope-rewritten path: the + // `claude_code.` span-name fingerprint is conjunctive with span.type + // co-evidence — with it the span classifies; without it (span names are + // customer data in other dialects: a crewai crew named "claude_code" yields + // a span literally named "claude_code.kickoff") the span stays unclaimed. + out.push(one( + NEUTRAL_SCOPE, + case( + "resolution/attr_only", + "attr_only/claude_span_name_with_span_type", + "scope-rewritten claude dialect: claude_code.* name + span.type classifies", + "claude_code.llm_request", + vec![s("span.type", "llm_request")], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "resolution/attr_only", + "attr_only/claude_span_name_without_span_type", + "the crewai customer-data hazard: claude_code.-named span without span.type", + "claude_code.kickoff", + vec![s("http.route", "/x")], + ), + )); + // Phase-2 effect_ai (queue E1–E3): the 6 ordinary-English Class.method names + // classify only alongside Effect-ecosystem evidence, the 7 module-path names + // stay standalone, and the bare-word pair is the guarded rename-proof tier. + out.push(one( + NEUTRAL_SCOPE, + case( + "resolution/attr_only", + "attr_only/effect_ai_guarded_name_without_evidence", + "an ordinary-English Class.method name with no Effect evidence stays unclaimed", + "Chat.export", + vec![s("http.route", "/x")], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "resolution/attr_only", + "attr_only/effect_ai_strong_name_without_evidence", + "a module-path name classifies without any Effect evidence (tier 1 unguarded)", + "EmbeddingModel.embed", + vec![s("http.route", "/x")], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "resolution/attr_only", + "attr_only/effect_ai_bare_words_without_evidence", + "toolChoice + concurrency alone (renamed spans, no Effect evidence) stay unclaimed", + "op", + vec![s("toolChoice", "undefined"), s("concurrency", "undefined")], + ), + )); + out.push( + Group::new( + NEUTRAL_SCOPE, + vec![case( + "resolution/attr_only", + "attr_only/effect_ai_bare_words_with_evidence", + "the E3 rename-proof tier: bare-word pair + the @effect/opentelemetry resource", + "op", + vec![s("toolChoice", "undefined"), s("concurrency", "undefined")], + )], + ) + .resource(vec![s("telemetry.sdk.name", "@effect/opentelemetry")]), + ); + // The tier-2 hazard, constructed rather than described: `scope_name == + // service.name` is the plain `getTracer(serviceName)` idiom — 100% of the + // openrouter capture satisfies it — so it must NOT carry an ordinary-English + // guarded name. The scope here is deliberately NOT Effect-branded; using + // "effect-ai-user" (as this case did) documents the clause without ever + // building the false positive it risks. + out.push( + Group::new( + "openrouter", + vec![case( + "resolution/attr_only", + "attr_only/effect_ai_guarded_name_via_scope_is_service", + "scope.name == service.name is the getTracer idiom, not Effect evidence: unclaimed", + "Toolkit.handle", + vec![s("tool", "get_weather")], + )], + ) + .resource(vec![s("service.name", "openrouter")]), + ); + out.push( + Group::new( + NEUTRAL_SCOPE, + vec![case( + "resolution/attr_only", + "attr_only/effect_ai_guarded_name_with_effect_resource", + "the tier-2 guard that survives: a guarded name under the @effect/opentelemetry resource", + "Toolkit.handle", + vec![s("tool", "get_weather")], + )], + ) + .resource(vec![s("telemetry.sdk.name", "@effect/opentelemetry")]), + ); + // Phase-2 spring_ai (fix-queue SP1): the tool-less ChatModel fallback is the + // POSITIVE conjunct `boot scope && gen_ai.system == "openai"`. Naive negation + // is banned — openrouter carries 1,477 gen_ai.system="openai" spans under its + // own scope and its FP ceiling is zero vendor claims. + out.push(one( + NEUTRAL_SCOPE, + case( + "resolution/attr_only", + "attr_only/spring_ai_openai_without_boot_scope", + "the openrouter trap: gen_ai.system=openai outside the Boot scope is not spring_ai", + "chat openai/gpt-4o-mini", + vec![s("gen_ai.system", "openai")], + ), + )); + out.push(one( + "org.springframework.boot", + case( + "resolution/attr_only", + "attr_only/spring_ai_openai_under_boot_scope", + "SP1: a tool-less ChatModel span (no spring.ai.* key) under the Boot scope", + "chat openai/gpt-4o-mini", + vec![s("gen_ai.system", "openai")], + ), + )); + out.push(one( + "org.springframework.boot", + case( + "resolution/attr_only", + "attr_only/spring_ai_boot_scope_http_span", + "the reason the Boot scope is conjunct-only: a plain Micrometer HTTP span", + "POST", + vec![s("http.request.method", "POST"), s("uri", "/v1/chat")], + ), + )); + out +} + +/// Spans carrying two vendors' evidence at once: the global priority ordering, not a +/// per-candidate probe, decides. +fn cross_vendor_groups() -> Vec { + vec![ + Group::new( + "openinference.instrumentation.openai", + vec![case( + "resolution/cross_vendor", + "cross_vendor/openai_scope_vs_crewai_attr", + "sufficient scope (band 3xxxx) outranks another vendor's attr hit (2xxxx)", + "ChatCompletion", + vec![ + s("openinference.span.kind", "LLM"), + s("task_key", "research"), + ], + )], + ), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "resolution/cross_vendor", + "cross_vendor/mastra_resource_vs_agno_attr", + "sufficient *resource* matcher outranks an attr hit", + "agent.generate", + vec![s("agno.run.id", "r-1")], + )], + ) + .resource(vec![ + s("service.name", "mastra-app"), + s("telemetry.sdk.name", "@mastra/otel-exporter"), + ]), + one( + NEUTRAL_SCOPE, + case( + "resolution/cross_vendor", + "cross_vendor/two_attr_bands", + "two vendors' attr matchers on one span: highest priority wins", + "op", + vec![ + s("spring.ai.kind", "chat_client"), + s("mastra.span.type", "llm"), + ], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/cross_vendor", + "cross_vendor/vendor_beats_unknown", + "phase 2 flipped this: vercel's ai. evidence is scope-gated, so under an \ + unclaimed scope the unknown tier wins (gen_ai.operation.name → unknown:genai)", + "ai.generateText", + vec![ + s("ai.operationId", "ai.generateText"), + s("gen_ai.operation.name", "chat"), + ], + ), + ), + one( + "org.springframework.boot", + case( + "resolution/cross_vendor", + "cross_vendor/unpromoted_candidate_loses_to_unknown", + "an unpromoted candidate does not suppress the unknown tier", + "POST", + vec![s("gen_ai.operation.name", "chat")], + ), + ), + one( + "gcp.vertex.agent", + case( + "resolution/cross_vendor", + "cross_vendor/sufficient_scope_with_foreign_system", + "sufficient scope wins over a conflicting gen_ai.system attr matcher", + "invocation", + vec![s("gen_ai.system", "langchain")], + ), + ), + ] +} + +fn unknown_tier_groups() -> Vec { + vec![ + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/genai", + "present(gen_ai.operation.name)", + "chat", + vec![s("gen_ai.operation.name", "chat")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/openinference", + "present(openinference.span.kind)", + "call", + vec![s("openinference.span.kind", "LLM")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/llm_prefix", + "key_prefix(llm.)", + "call", + vec![s("llm.model_name", "gpt-4o")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/traceloop_prefix", + "key_prefix(traceloop.)", + "workflow", + vec![s("traceloop.workflow.name", "w")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/ai_prefix", + "key_prefix(ai.) outside the AI SDK's ai/gen_ai scopes — reachable since \ + phase 2 scope-gated vercel's prefix evidence (fix-queue X3)", + "call", + vec![s("ai.telemetry.functionId", "f")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/co_occurrence_gate_off", + "input.value/output.value alone are deliberately not fingerprints", + "handler", + vec![s("input.value", "{}"), s("output.value", "{}")], + ), + ), + // S11 (owner decision 2026-08-13): the witness for the clause vercel LOST. + // Removing a rule leaves nothing behind to fail if it comes back, so the + // negative is written down: the AI SDK's own hardcoded tracer scope, holding + // a span with the GenAI-semconv key OTel requires on every such span and no + // `ai.*` at all, must NOT resolve to `vercel_ai_sdk`. That is exactly eve's + // default-config `chat` population — 53 corpus spans given up by choice. Its + // positive twin is `pseudo/scope_name_eq_is_case_sensitive`, the same scope + // WITH an `ai.*` key, which still classifies. + one( + "gen_ai", + case( + "resolution/unknown_tier", + "unknown/genai_in_the_ai_sdk_scope", + "S11: scope `gen_ai` + gen_ai.operation.name is not vendor evidence — \ + the conjunct is semconv-required, so the clause reduced to \"a tracer \ + named gen_ai claims everything inside it\" over customer data", + "chat openai/gpt-4o-mini", + vec![s("gen_ai.operation.name", "chat")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/co_occurrence_gate_on", + "the same generic values with an OpenInference attribute present", + "handler", + vec![ + s("input.value", "{}"), + s("openinference.span.kind", "CHAIN"), + ], + ), + ), + // Phase-2 X1/D-4: the other OpenInference spelling as co-evidence. The + // ordering inside the tier is what makes this unknown:openinference + // rather than the unknown:other the bare `llm.` rule would give it. + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/co_occurrence_gate_llm_namespace", + "generic output.value with the OpenInference llm.* namespace as co-evidence", + "handler", + vec![s("output.value", "{}"), s("llm.model_name", "gpt-4o")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/unknown_tier", + "unknown/priority_between_buckets", + "genai outranks openinference outranks llm.", + "call", + vec![ + s("gen_ai.operation.name", "chat"), + s("openinference.span.kind", "LLM"), + s("llm.model_name", "gpt-4o"), + ], + ), + ), + ] +} + +fn non_ai_groups() -> Vec { + vec![ + one( + "@opentelemetry/instrumentation-http", + case( + "resolution/non_ai", + "non_ai/http_server", + "ordinary HTTP server span", + "GET /health", + vec![ + s("http.request.method", "GET"), + s("url.path", "/health"), + int("http.response.status_code", 200), + ], + ), + ), + one( + "@opentelemetry/instrumentation-pg", + case( + "resolution/non_ai", + "non_ai/db_client", + "ordinary DB client span", + "SELECT maple.traces", + vec![ + s("db.system.name", "postgresql"), + s("db.namespace", "maple"), + ], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "resolution/non_ai", + "non_ai/no_attributes", + "no attributes at all", + "work", + vec![], + ), + ), + Group::new( + "", + vec![case( + "resolution/non_ai", + "non_ai/empty_scope_name", + "empty scope name", + "work", + vec![s("http.route", "/")], + )], + ), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "resolution/non_ai", + "non_ai/no_scope", + "ScopeSpans with no InstrumentationScope at all", + "work", + vec![s("http.route", "/")], + )], + ) + .no_scope(), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "resolution/non_ai", + "non_ai/no_resource", + "ResourceSpans with no resource attributes", + "work", + vec![s("http.route", "/")], + )], + ) + .resource(vec![]), + ] +} + +/// The session-state ladder (1..6), per vendor, one span per rung it can reach. +// One `push` per vendor keeps each vendor's ladder a self-contained block with its own +// comment; a single `vec![]` literal would bury the boundaries. +#[allow(clippy::vec_init_then_push)] +fn session_state_groups() -> Vec { + let mut out = Vec::new(); + + // -- agno: two candidates, authority = AGENT kind or an agno.workflow. key ---- + out.push(Group::new( + "openinference.instrumentation.agno", + vec![ + case( + "session/agno", + "session/agno/presence_gated_session_id", + "no AGENT kind, but session.id present — the phase-2 A3 presence branch \ + makes the session.id candidate authoritative (run.id's stays state 2)", + "Model.invoke", + vec![s("agno.run.id", "r-1"), s("session.id", "s-1")], + ), + case( + "session/agno", + "session/agno/state2_not_authoritative", + "no AGENT kind, no agno.workflow. key, no session.id — both candidates \ + report not-authoritative", + "Model.invoke", + vec![s("agno.run.id", "r-1")], + ), + case( + "session/agno", + "session/agno/state3_key_absent", + "authoritative, neither candidate key present", + "Agent.run", + vec![s("openinference.span.kind", "AGENT")], + ), + case( + "session/agno", + "session/agno/state4_empty", + "authoritative, session.id present but empty; agno.run.id also empty", + "Agent.run", + vec![ + s("openinference.span.kind", "AGENT"), + s("session.id", ""), + s("agno.run.id", ""), + ], + ), + case( + "session/agno", + "session/agno/state5_run_only", + "run-granularity candidate only", + "Agent.run", + vec![ + s("openinference.span.kind", "AGENT"), + s("agno.run.id", "r-9"), + ], + ), + case( + "session/agno", + "session/agno/state6_session", + "session-granularity candidate resolves; max over candidates", + "Agent.run", + vec![ + s("openinference.span.kind", "AGENT"), + s("session.id", "sess-agno"), + s("agno.run.id", "r-9"), + ], + ), + case( + "session/agno", + "session/agno/state6_via_workflow_prefix", + "authority via key_prefix(agno.workflow.)", + "Workflow.run", + vec![ + s("agno.workflow.name", "research"), + s("session.id", "sess-agno-2"), + ], + ), + case( + "session/agno", + "session/agno/state4_empty_wins_over_absent", + "empty session.id (4) beats absent agno.run.id (3) under max", + "Agent.run", + vec![s("openinference.span.kind", "AGENT"), s("session.id", "")], + ), + ], + )); + + // -- claude_agent_sdk: authority = present(span.type) ------------------------- + out.push(Group::new( + "com.anthropic.claude_code", + vec![ + case( + "session/claude_agent_sdk", + "session/claude/state2", + "no span.type ⇒ neither candidate is authoritative", + "claude_code.other", + vec![s("session.id", "sess-1")], + ), + case( + "session/claude_agent_sdk", + "session/claude/state3", + "authoritative, no key", + "claude_code.interaction", + vec![s("span.type", "interaction")], + ), + case( + "session/claude_agent_sdk", + "session/claude/state4", + "present-but-empty session.id", + "claude_code.interaction", + vec![s("span.type", "interaction"), s("session.id", "")], + ), + case( + "session/claude_agent_sdk", + "session/claude/state5_user_only", + "user granularity only", + "claude_code.interaction", + vec![s("span.type", "interaction"), s("user.id", "u-7")], + ), + case( + "session/claude_agent_sdk", + "session/claude/state6", + "session id present and valid", + "claude_code.interaction", + vec![ + s("span.type", "interaction"), + s("session.id", "sess-claude"), + s("user.id", "u-7"), + ], + ), + ], + )); + + // -- flue: one ALWAYS candidate, one gated; decoy value 'default' ------------- + out.push(Group::new( + "@flue/opentelemetry", + vec![ + case( + "session/flue", + "session/flue/state3_always_candidate_absent", + "the ALWAYS candidate is authoritative but its key is absent", + "flue.tool", + vec![s("flue.operation.kind", "tool")], + ), + case( + "session/flue", + "session/flue/state5_instance", + "instance granularity resolves at 5", + "flue.tool", + vec![ + s("flue.operation.kind", "tool"), + s("flue.instance.id", "inst-7"), + ], + ), + case( + "session/flue", + "session/flue/state6_conversation", + "prompt span with a conversation id", + "flue.prompt", + vec![ + s("flue.operation.kind", "prompt"), + s("gen_ai.conversation.id", "conv-3"), + s("flue.instance.id", "inst-7"), + ], + ), + case( + "session/flue", + "session/flue/state5_decoy_conversation", + "decoy value 'default' invalidates candidate 1 (4) but instance still resolves (5)", + "flue.prompt", + vec![ + s("flue.operation.kind", "prompt"), + s("gen_ai.conversation.id", "default"), + s("flue.instance.id", "inst-7"), + ], + ), + case( + "session/flue", + "session/flue/state4_decoy_only", + "decoy conversation id, no instance id ⇒ 4", + "flue.prompt", + vec![ + s("flue.operation.kind", "prompt"), + s("gen_ai.conversation.id", "default"), + ], + ), + // Phase-2 F1: a delegate whose operation_start dedup guard missed — + // kind=prompt AND flue.task.id AND its own SUB-conversation id. The + // authority rejects it, so it resolves at instance granularity (5) + // instead of minting a sub-conversation session (6). + case( + "session/flue", + "session/flue/state5_delegate_prompt_rejected", + "F1: a kind=prompt span carrying flue.task.id is not session-authoritative", + "flue.prompt", + vec![ + s("flue.operation.kind", "prompt"), + s("flue.task.id", "task_01KZ"), + s("gen_ai.conversation.id", "conv-delegate-9"), + s("flue.instance.id", "inst-7"), + ], + ), + ], + )); + + // -- google_adk: three candidates, disjoint authority populations ------------- + out.push(Group::new( + "gcp.vertex.agent", + vec![ + case( + "session/google_adk", + "session/adk/state2", + "none of the three authority predicates hold", + "internal", + vec![s("gcp.vertex.agent.session_id", "s-1")], + ), + case( + "session/google_adk", + "session/adk/state3", + "authoritative via gen_ai.system, key absent", + "invocation", + vec![s("gen_ai.system", "gcp.vertex.agent")], + ), + case( + "session/google_adk", + "session/adk/state6_via_system", + "candidate 2 resolves at session granularity", + "invocation", + vec![ + s("gen_ai.system", "gcp.vertex.agent"), + s("gcp.vertex.agent.session_id", "s-1"), + ], + ), + case( + "session/google_adk", + "session/adk/state6_via_conversation", + "candidate 1's population: invoke_agent", + "invoke_agent weather", + vec![ + s("gen_ai.operation.name", "invoke_agent"), + s("gen_ai.conversation.id", "c-9"), + ], + ), + case( + "session/google_adk", + "session/adk/state6_via_invoke_workflow", + "phase-2 G2: the adk-schema-v2 root (source-cited, zero corpus spans) \ + is authoritative for candidate 1", + "invoke_workflow entrypoint", + vec![ + s("gen_ai.operation.name", "invoke_workflow"), + s("gen_ai.conversation.id", "c-10"), + ], + ), + case( + "session/google_adk", + "session/adk/state5_invocation_run", + "run-granularity candidate 3 (authority = present(gen_ai.request.model))", + "generate_content", + vec![ + s("gen_ai.request.model", "gemini-2.0-flash"), + s("gcp.vertex.agent.invocation_id", "inv-1"), + ], + ), + case( + "session/google_adk", + "session/adk/state6_max_unions_disjoint", + "one candidate at 2, one at 6 — max unions instead of cancelling", + "generate_content", + vec![ + s("gen_ai.operation.name", "generate_content"), + s("gen_ai.conversation.id", "c-11"), + s("gcp.vertex.agent.invocation_id", "inv-2"), + ], + ), + ], + )); + + // -- spring_ai: promoted candidate + decoy 'default' -------------------------- + out.push( + Group::new( + "org.springframework.boot", + vec![ + case( + "session/spring_ai", + "session/spring/state2", + "not a chat_client span ⇒ not authoritative", + "chat", + vec![ + s("spring.ai.kind", "chat_model"), + s("spring.ai.chat.client.conversation.id", "c-1"), + ], + ), + case( + "session/spring_ai", + "session/spring/state3", + "authoritative, key absent", + "chat_client", + vec![s("spring.ai.kind", "chat_client")], + ), + case( + "session/spring_ai", + "session/spring/state4_decoy", + "the 'default' decoy value", + "chat_client", + vec![ + s("spring.ai.kind", "chat_client"), + s("spring.ai.chat.client.conversation.id", "default"), + ], + ), + case( + "session/spring_ai", + "session/spring/state4_empty", + "present-but-empty", + "chat_client", + vec![ + s("spring.ai.kind", "chat_client"), + s("spring.ai.chat.client.conversation.id", ""), + ], + ), + case( + "session/spring_ai", + "session/spring/state6", + "resolved", + "chat_client", + vec![ + s("spring.ai.kind", "chat_client"), + s("spring.ai.chat.client.conversation.id", "conv-42"), + ], + ), + ], + ) + .resource(vec![s("service.name", "spring-ai-app")]), + ); + + // -- litellm: two user-granularity candidates, decoys incl. the empty string -- + out.push(Group::new( + "litellm", + vec![ + case( + "session/litellm", + "session/litellm/state2", + "no litellm.call_id ⇒ not authoritative", + "litellm_request", + vec![s("metadata.user_api_key_user_id", "u-1")], + ), + case( + "session/litellm", + "session/litellm/state3", + "authoritative, no key", + "litellm_request", + vec![s("litellm.call_id", "c-1")], + ), + case( + "session/litellm", + "session/litellm/state4_decoy_default_user", + "'default_user_id' is a decoy value", + "litellm_request", + vec![ + s("litellm.call_id", "c-1"), + s("metadata.user_api_key_end_user_id", "default_user_id"), + ], + ), + case( + "session/litellm", + "session/litellm/state4_decoy_empty", + "the empty string is BOTH a non_empty failure and a declared decoy", + "litellm_request", + vec![ + s("litellm.call_id", "c-1"), + s("metadata.user_api_key_end_user_id", ""), + ], + ), + case( + "session/litellm", + "session/litellm/state5", + "user granularity resolves at 5, never 6", + "litellm_request", + vec![ + s("litellm.call_id", "c-1"), + s("metadata.user_api_key_user_id", "user-77"), + ], + ), + ], + )); + + // -- langchain: ALWAYS candidate + decoy -------------------------------------- + out.push(Group::new( + "langsmith", + vec![ + case( + "session/langchain", + "session/langchain/state3", + "ALWAYS-authoritative candidate with no key", + "chain", + vec![s("langsmith.trace.name", "chain")], + ), + case( + "session/langchain", + "session/langchain/state4_decoy", + "'default' decoy", + "chain", + vec![s("langsmith.metadata.thread_id", "default")], + ), + case( + "session/langchain", + "session/langchain/state6", + "resolved thread id", + "chain", + vec![s("langsmith.metadata.thread_id", "thread-9")], + ), + ], + )); + + // -- mastra: three candidates at three granularities -------------------------- + out.push(Group::new( + "@mastra/otel-exporter", + vec![ + case( + "session/mastra", + "session/mastra/state2", + "no mastra.span.type ⇒ not authoritative", + "agent.generate", + vec![s("gen_ai.conversation.id", "c-1")], + ), + case( + "session/mastra", + "session/mastra/state3", + "authoritative, no candidate key", + "agent.generate", + vec![s("mastra.span.type", "agent_run")], + ), + case( + "session/mastra", + "session/mastra/state5_run", + "run granularity", + "agent.generate", + vec![ + s("mastra.span.type", "agent_run"), + s("mastra.metadata.runId", "run-1"), + ], + ), + case( + "session/mastra", + "session/mastra/state5_user", + "user granularity", + "agent.generate", + vec![ + s("mastra.span.type", "agent_run"), + s("mastra.metadata.resourceId", "res-1"), + ], + ), + case( + "session/mastra", + "session/mastra/state6_max", + "all three candidates resolve; max picks session and the hash follows it", + "agent.generate", + vec![ + s("mastra.span.type", "agent_run"), + s("gen_ai.conversation.id", "conv-mastra"), + s("mastra.metadata.runId", "run-1"), + s("mastra.metadata.resourceId", "res-1"), + ], + ), + ], + )); + + // -- pydantic_ai -------------------------------------------------------------- + out.push(Group::new( + "pydantic-ai", + vec![ + case( + "session/pydantic_ai", + "session/pydantic/state2", + "no gen_ai.operation.name ⇒ not authoritative", + "agent run", + vec![s("gen_ai.conversation.id", "c-1")], + ), + case( + "session/pydantic_ai", + "session/pydantic/state5_call_id", + "run granularity only", + "agent run", + vec![ + s("gen_ai.operation.name", "invoke_agent"), + s("gen_ai.agent.call.id", "call-1"), + ], + ), + case( + "session/pydantic_ai", + "session/pydantic/state6", + "conversation id at session granularity", + "agent run", + vec![ + s("gen_ai.operation.name", "invoke_agent"), + s("gen_ai.conversation.id", "run-1"), + s("pydantic_ai.all_messages", "[]"), + ], + ), + case( + "session/pydantic_ai", + "session/pydantic/state5_uuid7_demotion", + "phase-2 P1: a strict-UUIDv7 conversation id is pydantic's auto-minted \ + per-run default — value-conditional granularity demotes it to run (5); \ + the hash still comes from the conversation id (candidate-order tie)", + "agent run", + vec![ + s("gen_ai.operation.name", "invoke_agent"), + s( + "gen_ai.conversation.id", + "01890a5d-ac96-774b-bcce-b302099a8057", + ), + s("gen_ai.agent.call.id", "0195e2f1-7d13-7cc0-a583-53c804a45f92"), + ], + ), + ], + )); + + // -- microsoft_agent_framework: two decoy values ------------------------------ + out.push(Group::new( + "agent_framework", + vec![ + case( + "session/microsoft_agent_framework", + "session/maf/state2", + "operation is not invoke_agent", + "chat", + vec![s("gen_ai.conversation.id", "c-1")], + ), + case( + "session/microsoft_agent_framework", + "session/maf/state4_decoy_local_history", + "'agent_framework_local_history_persistence' decoy", + "invoke_agent", + vec![ + s("gen_ai.operation.name", "invoke_agent"), + s( + "gen_ai.conversation.id", + "agent_framework_local_history_persistence", + ), + ], + ), + case( + "session/microsoft_agent_framework", + "session/maf/state4_decoy_unknown", + "'unknown' decoy", + "invoke_agent", + vec![ + s("gen_ai.operation.name", "invoke_agent"), + s("gen_ai.conversation.id", "unknown"), + ], + ), + case( + "session/microsoft_agent_framework", + "session/maf/state6", + "resolved", + "invoke_agent", + vec![ + s("gen_ai.operation.name", "invoke_agent"), + s("gen_ai.conversation.id", "thread_abc"), + ], + ), + ], + )); + + // -- strands / openai_agents_sdk / smolagents / crewai / dspy / llamaindex ---- + out.push(Group::new( + "strands.telemetry.tracer", + vec![ + case( + "session/strands", + "session/strands/state2", + "no gen_ai.system / provider ⇒ not authoritative", + "Model invoke", + vec![s("session.id", "s-1")], + ), + case( + "session/strands", + "session/strands/state6", + "authoritative via gen_ai.system", + "Cycle", + vec![s("gen_ai.system", "strands-agents"), s("session.id", "s-2")], + ), + case( + "session/strands", + "session/strands/state6_via_provider", + "authoritative via gen_ai.provider.name", + "Cycle", + vec![ + s("gen_ai.provider.name", "strands-agents"), + s("session.id", "s-3"), + ], + ), + ], + )); + out.push(Group::new( + "openinference.instrumentation.openai_agents", + vec![ + case( + "session/openai_agents_sdk", + "session/openai_agents/state2", + "no openinference.span.kind ⇒ not authoritative", + "Response", + vec![s("session.id", "s-1")], + ), + case( + "session/openai_agents_sdk", + "session/openai_agents/state6_session_id", + "session.id wins; gen_ai.conversation.id ties at 6 and loses on order", + "Agent workflow", + vec![ + s("openinference.span.kind", "AGENT"), + s("session.id", "sess-a"), + s("gen_ai.conversation.id", "conv-b"), + ], + ), + case( + "session/openai_agents_sdk", + "session/openai_agents/state6_conversation_only", + "second candidate alone", + "Agent workflow", + vec![ + s("openinference.span.kind", "AGENT"), + s("gen_ai.conversation.id", "conv-b"), + ], + ), + ], + )); + out.push(Group::new( + "openinference.instrumentation.smolagents", + vec![ + case( + "session/smolagents", + "session/smolagents/state3", + "authoritative, no key", + "CodeAgent.run", + vec![s("openinference.span.kind", "AGENT")], + ), + case( + "session/smolagents", + "session/smolagents/state5_user", + "user granularity only", + "CodeAgent.run", + vec![s("openinference.span.kind", "AGENT"), s("user.id", "u-1")], + ), + ], + )); + out.push(Group::new( + "openinference.instrumentation.crewai", + vec![ + case( + "session/crewai", + "session/crewai/state3", + "authority is the scope itself; key absent", + "Crew.kickoff", + vec![s("crew_key", "k")], + ), + case( + "session/crewai", + "session/crewai/state6", + "resolved", + "Crew.kickoff", + vec![s("crew_key", "k"), s("session.id", "sess-crew")], + ), + ], + )); + out.push(Group::new( + "openinference.instrumentation.dspy", + vec![case( + "session/dspy", + "session/dspy/state6", + "scope-gated authority, session granularity", + "Predict.forward", + vec![s("session.id", "sess-dspy"), s("user.id", "u-dspy")], + )], + )); + out.push(Group::new( + "llamaindex.opentelemetry.tracer", + vec![ + case( + "session/llamaindex", + "session/llamaindex/state3", + "ALWAYS candidate, key absent", + "query", + vec![s("llamaindex.span.kind", "query")], + ), + case( + "session/llamaindex", + "session/llamaindex/state5", + "run granularity", + "query", + vec![s("llamaindex.run_id", "run-3")], + ), + ], + )); + + // -- vendors with no session rules at all ⇒ state 1 --------------------------- + for (scope_name, label) in [ + ("haystack", "haystack"), + ( + "semantic_kernel.functions.kernel_function", + "semantic_kernel", + ), + ("ai", "vercel_ai_sdk"), + ] { + out.push(one( + scope_name, + case( + "session/no_rules", + &format!("session/no_rules/{label}"), + "vendor with zero session candidates ⇒ state 1", + "op", + vec![s("ai.operationId", "ai.generateText")], + ), + )); + } + out.push(one( + NEUTRAL_SCOPE, + case( + "session/no_rules", + "session/no_rules/unknown_bucket", + "unknown-tier buckets carry no session rules ⇒ state 1", + "call", + vec![s("gen_ai.operation.name", "chat"), s("session.id", "s-1")], + ), + )); + out.push(one( + "effect_ai_scope_is_not_a_thing", + case( + "session/no_rules", + "session/no_rules/effect_ai", + "effect_ai classifies on span name and has no candidates", + "LanguageModel.generateText", + vec![s("gen_ai.operation.name", "chat")], + ), + )); + + out +} + +/// Typed `AnyValue`s: the canonicalization the alignment contract rests on. Each case +/// pairs a typed value with a rule that reads it, so a canonicalization difference +/// changes the verdict rather than hiding in an unread column. +fn typed_value_groups() -> Vec { + let mut out = vec![ + // bool ⇒ "true"/"false", read by langchain's resource matcher. + Group::new( + NEUTRAL_SCOPE, + vec![case( + "values/typed", + "typed/bool_true_resource", + "BoolValue(true) canonicalizes to 'true' for eq()", + "chain", + vec![s("langsmith.trace.name", "x")], + )], + ) + .resource(vec![ + s("service.name", "lc"), + boolean("langsmith.internal_provider", true), + ]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "values/typed", + "typed/bool_false_resource", + "BoolValue(false) must NOT satisfy eq(..., 'true')", + "chain", + vec![s("http.route", "/x")], + )], + ) + .resource(vec![ + s("service.name", "lc"), + boolean("langsmith.internal_provider", false), + ]), + ]; + + // Typed session-key values: the hash is taken over the canonical string. + let typed_keys: Vec<(&str, KeyValue, &str)> = vec![ + ( + "int", + int("session.id", 4_294_967_296), + "IntValue beyond 2^32 as a session key", + ), + ( + "int_negative", + int("session.id", i64::MIN), + "IntValue::MIN as a session key", + ), + ( + "double", + double("session.id", 1.5), + "DoubleValue canonicalizes with Rust's float formatting", + ), + ( + "double_integral", + double("session.id", 42.0), + "42.0 renders as '42', not '42.0'", + ), + ( + "bool", + boolean("session.id", true), + "BoolValue as a session key", + ), + ( + "bytes", + bytes("session.id", &[0xde, 0xad, 0xbe, 0xef, 0x00, 0x01]), + "BytesValue hex-encodes (the row writer's rule, not JSON.stringify)", + ), + ( + "array", + array( + "session.id", + vec![s("", "a"), int("", 2), boolean("", false)], + ), + "ArrayValue renders as a JSON array of canonical strings", + ), + ( + "kvlist", + kvlist( + "session.id", + vec![s("z", "last"), s("a", "first"), int("m", 7)], + ), + "KvlistValue renders as a JSON object; the row Map's key order is what SQL sees", + ), + ( + "nested", + array( + "session.id", + vec![kvlist("", vec![s("k", "v")]), array("", vec![int("", 1)])], + ), + "nested array/kvlist", + ), + ( + "untyped", + untyped("session.id"), + "AnyValue with no value ⇒ empty string ⇒ state 4", + ), + ( + "valueless", + valueless("session.id"), + "KeyValue with no AnyValue ⇒ empty string ⇒ state 4", + ), + ]; + for (label, key_value, note) in typed_keys { + out.push(Group::new( + "com.anthropic.claude_code", + vec![case( + "values/typed", + &format!("typed/session_key_{label}"), + note, + "claude_code.interaction", + vec![s("span.type", "interaction"), key_value], + )], + )); + } + + // Typed values on keys compared by eq(): a canonicalization slip flips the vendor. + out.push(one( + NEUTRAL_SCOPE, + case( + "values/typed", + "typed/eq_int_vs_string", + "IntValue on gen_ai.system cannot match any vendor's string literal", + "op", + vec![int("gen_ai.system", 42)], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "values/typed", + "typed/eq_kvlist_on_matched_key", + "KvlistValue on an eq()-compared key", + "op", + vec![kvlist("gen_ai.system", vec![s("spring_ai", "yes")])], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "values/typed", + "typed/present_only_key_is_type_blind", + "present() ignores the value's type entirely", + "op", + vec![bytes("gen_ai.operation.name", &[0x00, 0xff])], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "values/typed", + "typed/array_on_prefix_key", + "key_prefix() ignores values; the key alone decides", + "op", + vec![array("llm.token_counts", vec![int("", 1), int("", 2)])], + ), + )); + out +} + +/// Present-but-empty: the algebra's load-bearing subtlety. `mapContains` sees it, +/// `!= ''` would not. +fn present_but_empty_groups() -> Vec { + vec![ + one( + NEUTRAL_SCOPE, + case( + "values/present_empty", + "present_empty/unknown_genai", + "gen_ai.operation.name = '' still fingerprints as unknown:genai", + "llm", + vec![s("gen_ai.operation.name", "")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "values/present_empty", + "present_empty/unknown_openinference", + "openinference.span.kind = ''", + "llm", + vec![s("openinference.span.kind", "")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "values/present_empty", + "present_empty/eq_matcher_not_satisfied", + "an empty value cannot satisfy eq() against a non-empty literal", + "op", + vec![s("gen_ai.system", "")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "values/present_empty", + "present_empty/prefix_key_empty_value", + "key_prefix() reads keys, so an empty value still hits", + "op", + vec![s("spring.ai.kind", "")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "values/present_empty", + "present_empty/empty_key_name", + "an attribute whose key is the empty string", + "op", + vec![s("", ""), s("gen_ai.operation.name", "chat")], + ), + ), + Group::new( + "com.anthropic.claude_code", + vec![ + case( + "values/present_empty", + "present_empty/state4_vs_state3_empty", + "present-but-empty ⇒ 4", + "claude_code.interaction", + vec![s("span.type", "interaction"), s("session.id", "")], + ), + case( + "values/present_empty", + "present_empty/state4_vs_state3_absent", + "the same span with the key absent ⇒ 3", + "claude_code.interaction", + vec![s("span.type", "interaction")], + ), + case( + "values/present_empty", + "present_empty/authority_key_empty", + "the authority predicate is present(), so an empty span.type still grants it", + "claude_code.interaction", + vec![s("span.type", ""), s("session.id", "sess-empty-auth")], + ), + ], + ), + one( + "@flue/opentelemetry", + case( + "values/present_empty", + "present_empty/eq_authority_empty", + "eq()-based authority against an empty value", + "flue.prompt", + vec![ + s("flue.operation.kind", ""), + s("gen_ai.conversation.id", "c-1"), + ], + ), + ), + ] +} + +/// Duplicate keys. Rule-referenced keys are first-occurrence-wins inside the +/// classifier; the row Map keeps the last occurrence for every key (the v1 +/// row-writer coupling that forced the Map to agree with the matcher is gone). +fn duplicate_key_groups() -> Vec { + let mut out = Vec::new(); + let orders: [(&str, [&str; 2]); 2] = [ + ("spring_first", ["spring_ai", "strands-agents"]), + ("strands_first", ["strands-agents", "spring_ai"]), + ]; + for (label, [first, second]) in orders { + out.push(one( + NEUTRAL_SCOPE, + case( + "keys/duplicate", + &format!("duplicate/gen_ai_system_{label}"), + "duplicate registry key: the first occurrence decides both the verdict and the row", + "chat", + vec![s("gen_ai.system", first), s("gen_ai.system", second)], + ), + )); + } + out.push(Group::new( + "com.anthropic.claude_code", + vec![ + case( + "keys/duplicate", + "duplicate/session_id_valid_then_empty", + "first occurrence valid, second empty ⇒ state 6", + "claude_code.interaction", + vec![ + s("span.type", "interaction"), + s("session.id", "sess-dup-1"), + s("session.id", ""), + ], + ), + case( + "keys/duplicate", + "duplicate/session_id_empty_then_valid", + "first occurrence empty, second valid ⇒ state 4", + "claude_code.interaction", + vec![ + s("span.type", "interaction"), + s("session.id", ""), + s("session.id", "sess-dup-2"), + ], + ), + case( + "keys/duplicate", + "duplicate/session_id_typed_then_string", + "typed first occurrence wins over a later string", + "claude_code.interaction", + vec![ + s("span.type", "interaction"), + int("session.id", 7), + s("session.id", "nine"), + ], + ), + case( + "keys/duplicate", + "duplicate/authority_key_duplicated", + "the authority key duplicated with a contradicting second value", + "claude_code.interaction", + vec![ + s("span.type", "interaction"), + s("span.type", "not-a-type"), + s("session.id", "sess-dup-3"), + ], + ), + case( + "keys/duplicate", + "duplicate/authority_key_contradiction_first", + "the mirror image: the non-matching value comes first", + "claude_code.interaction", + vec![ + s("span.type", "not-a-type"), + s("span.type", "interaction"), + s("session.id", "sess-dup-4"), + ], + ), + case( + "keys/duplicate", + "duplicate/triplicate", + "three occurrences of a registry key", + "claude_code.interaction", + vec![ + s("span.type", "interaction"), + s("session.id", "first"), + s("session.id", "second"), + s("session.id", "third"), + ], + ), + ], + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "keys/duplicate", + "duplicate/non_registry_key_last_wins", + "a key no rule consults keeps the LAST occurrence in the row Map", + "op", + vec![ + s("http.route", "/first"), + s("http.route", "/second"), + s("gen_ai.operation.name", "chat"), + ], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "keys/duplicate", + "duplicate/prefix_family_duplicated", + "duplicated keys inside a key_prefix family", + "op", + vec![ + s("spring.ai.kind", "chat_client"), + s("spring.ai.kind", "chat_model"), + s("spring.ai.other", "x"), + ], + ), + )); + out.push( + Group::new( + NEUTRAL_SCOPE, + vec![case( + "keys/duplicate", + "duplicate/resource_and_scope_same_key", + "the same registry key on the resource, the scope and the span", + "op", + vec![s("gen_ai.system", "spring_ai")], + )], + ) + .resource(vec![ + s("service.name", "dup"), + s("gen_ai.system", "strands-agents"), + ]) + .scope_attrs(vec![s("gen_ai.system", "langchain")]), + ); + out.push( + Group::new( + NEUTRAL_SCOPE, + vec![case( + "keys/duplicate", + "duplicate/scope_attribute_duplicated", + "duplicate keys inside the scope attribute list", + "op", + vec![s("http.route", "/x")], + )], + ) + .scope_attrs(vec![ + s("gen_ai.system", "spring_ai"), + s("gen_ai.system", "strands-agents"), + ]), + ); + out +} + +/// Near misses: keys engineered to share a first byte and length with a registry key, +/// or to sit one character away from a registry prefix. These are what the byte/length +/// screens are for, and a screen that leaks would show up as a spurious vendor. +fn near_miss_groups() -> Vec { + let near: Vec<(&str, &str)> = vec![ + ("gen_ai.systen", "same length, last byte differs"), + ("gen_ai.systemm", "one byte longer"), + ("gen_ai.syste", "one byte shorter"), + ("Gen_ai.system", "case-flipped first byte"), + ("gen_ai.operation.namf", "same length as a fingerprint key"), + ("session.iD", "case-flipped tail"), + ("sessionXid", "separator replaced"), + ("span.typ", "prefix of a registry key"), + ("span.typee", "registry key plus a byte"), + ("spring.ai", "the prefix without its trailing dot"), + ("spring/ai.kind", "separator swapped inside a prefix"), + ("spring.ao.kind", "one byte inside the prefix differs"), + ("agno", "prefix minus the dot"), + ("agnos.run", "prefix plus a byte before the dot"), + ("ai", "the shortest prefix minus its dot"), + ("aix.thing", "shares the first byte of the ai. prefix"), + ("llm", "llm. minus the dot"), + ("llmx.model", "llm prefix near miss"), + ("traceloop", "traceloop. minus the dot"), + ("traceloo.x", "one byte short inside the prefix"), + ("flue", "flue. minus the dot"), + ("mastra", "mastra. minus the dot"), + ("model_i", "resource key minus a byte"), + ("model_idx", "resource key plus a byte"), + ("telemetry.sdk.nam", "sufficient-resource key minus a byte"), + ("openinference.span.kin", "fingerprint key minus a byte"), + ("crew", "crew_ prefix minus the underscore"), + ("flow", "flow_ prefix minus the underscore"), + ("event_loop", "event_loop. minus the dot"), + ("executor", "executor. minus the dot"), + ("edge_group", "edge_group. minus the dot"), + ("sk", "sk. minus the dot"), + ("smolagents", "smolagents. minus the dot"), + ("langsmith", "langsmith. minus the dot"), + ("litellm", "litellm. minus the dot"), + ("haystack", "haystack. minus the dot"), + ("llamaindex", "llamaindex. minus the dot"), + ("pydantic_ai", "pydantic_ai. minus the dot"), + ("agent_framework", "agent_framework. minus the dot"), + ("gcp.vertex.agent", "gcp.vertex.agent. minus the dot"), + ]; + let mut out: Vec = near + .into_iter() + .map(|(key, note)| { + one( + NEUTRAL_SCOPE, + case( + "keys/near_miss", + &format!("near_miss/{key}"), + note, + "op", + vec![s(key, "value"), s("http.route", "/x")], + ), + ) + }) + .collect(); + + // Near-miss scope names against `eq(scope.name, …)` matchers. + for (scope_name, note) in [ + ("openinference.instrumentation.agn", "one byte short"), + ("openinference.instrumentation.agnoo", "one byte long"), + ("Openinference.instrumentation.agno", "case-flipped"), + ("openinference.instrumentation.agno ", "trailing space"), + (" openinference.instrumentation.agno", "leading space"), + ("pydantic_ai", "underscore instead of the hyphen"), + ("langsmit", "one byte short"), + ("litellmx", "one byte long"), + ] { + out.push(one( + scope_name, + case( + "keys/near_miss", + &format!("near_miss/scope/{scope_name}"), + note, + "op", + vec![s("http.route", "/x")], + ), + )); + } + out +} + +/// Unicode, including multi-byte boundaries — the byte screens index by first *byte*, +/// so a multi-byte lead byte is the interesting case. +fn unicode_groups() -> Vec { + let mut out = Vec::new(); + let unicode_values: Vec<(&str, String, &str)> = vec![ + ("emoji", "🙂🧠🚀".to_string(), "astral-plane characters"), + ("cjk", "会話-識別子-42".to_string(), "3-byte sequences"), + ("rtl", "מזהה-שיחה".to_string(), "RTL text"), + ( + "combining", + "e\u{0301}\u{0327}session".to_string(), + "combining marks after ASCII", + ), + ( + "zero_width", + "sess\u{200b}\u{200d}ion".to_string(), + "zero-width joiners inside the value", + ), + ( + "bom", + "\u{feff}session-1".to_string(), + "a leading byte-order mark", + ), + ( + "surrogate_pair_boundary", + format!("{}{}", "a".repeat(3), '\u{10FFFF}'), + "the last valid scalar value", + ), + ( + "nul_adjacent", + "before\u{0}after".to_string(), + "an embedded NUL", + ), + ("nul_leading", "\u{0}leading".to_string(), "a leading NUL"), + ( + "nul_trailing", + "trailing\u{0}".to_string(), + "a trailing NUL", + ), + ( + "control_chars", + "a\u{1}\u{2}\u{1f}b".to_string(), + "C0 control characters", + ), + ( + "quotes_and_backslash", + "it's a \\ \"test\" -- /* */".to_string(), + "SQL-hostile punctuation in a hashed value", + ), + ( + "newlines", + "line1\nline2\r\nline3\t".to_string(), + "newlines and tabs (NDJSON is line-delimited)", + ), + ]; + for (label, value, note) in unicode_values { + out.push(Group::new( + "com.anthropic.claude_code", + vec![case( + "unicode", + &format!("unicode/session_value/{label}"), + note, + "claude_code.interaction", + vec![s("span.type", "interaction"), s("session.id", &value)], + )], + )); + } + + // Unicode in KEYS: near-misses against the byte screen at multi-byte boundaries. + for (key, note) in [ + ("🌍.emoji.key", "a 4-byte lead byte"), + ("gen_ai.系统", "multi-byte tail on a registry-ish key"), + ("ai.🙂", "multi-byte tail inside a registry prefix"), + ("аi.operationId", "Cyrillic 'а' homoglyph as the first byte"), + ("gen_ai.system\u{0}", "a trailing NUL in the key"), + ("\u{feff}gen_ai.system", "a BOM in front of a registry key"), + ] { + out.push(one( + NEUTRAL_SCOPE, + case( + "unicode", + &format!("unicode/key/{key}"), + note, + "op", + vec![s(key, "v"), s("http.route", "/x")], + ), + )); + } + + // Unicode in the scope name and the span name (real columns on both sides). + out.push(one( + "openinference.instrumentation.agnő", + case( + "unicode", + "unicode/scope_name", + "a multi-byte near miss on a sufficient scope matcher", + "Agent.run", + vec![s("openinference.span.kind", "AGENT")], + ), + )); + out.push(one( + NEUTRAL_SCOPE, + case( + "unicode", + "unicode/span_name", + "a span name that is a unicode near miss of an effect_ai matcher", + "LanguageModel.generateTéxt", + vec![s("telemetry.sdk.name", "@effect/opentelemetry")], + ), + )); + + // The hash is over bytes, not chars: a non-ASCII session-key value. + out.push(Group::new( + "com.anthropic.claude_code", + vec![case( + "unicode", + "unicode/session_value", + "a non-ASCII session-key value", + "claude_code.interaction", + vec![ + s("span.type", "interaction"), + s("session.id", "セッション-1"), + ], + )], + )); + out +} + +/// Oversized values and attribute lists: the never-truncate list for registry keys, +/// and the spill path out of the classifier's inline attribute view. +fn oversized_groups() -> Vec { + let big_value = "x".repeat(64 * 1024); + let big_unicode = "🙂".repeat(4096); + let big_key = format!("spring.ai.{}", "k".repeat(4096)); + let mut filler: Vec = (0..60) + .map(|index| s(&format!("http.request.header.x_{index:03}"), "v")) + .collect(); + filler.push(s("span.type", "interaction")); + filler.push(s("session.id", "sess-wide")); + + // More registry-referenced keys than the classifier's inline view holds. + let mut spilled: Vec = vec![ + s("span.type", "interaction"), + s("session.id", "sess-spill"), + s("user.id", "u-spill"), + s("gen_ai.operation.name", "chat"), + s("gen_ai.system", "spring_ai"), + s("gen_ai.conversation.id", "conv-spill"), + s("gen_ai.request.model", "gpt-4o"), + s("openinference.span.kind", "AGENT"), + s("agno.run.id", "r-spill"), + s("mastra.span.type", "agent_run"), + s("mastra.metadata.runId", "run-spill"), + s("litellm.call_id", "call-spill"), + ]; + spilled.push(s("task_key", "research")); + + vec![ + Group::new( + "com.anthropic.claude_code", + vec![ + case( + "oversized", + "oversized/64kib_session_value", + "a 64 KiB session key value, hashed in full", + "claude_code.interaction", + vec![s("span.type", "interaction"), s("session.id", &big_value)], + ), + case( + "oversized", + "oversized/16kib_unicode_session_value", + "16 Ki astral characters (64 KiB of UTF-8)", + "claude_code.interaction", + vec![s("span.type", "interaction"), s("session.id", &big_unicode)], + ), + case( + "oversized", + "oversized/wide_attribute_list", + "60 non-registry attributes around the two that matter", + "claude_code.interaction", + filler, + ), + case( + "oversized", + "oversized/spilled_registry_keys", + "more registry keys than the inline attribute view holds", + "claude_code.interaction", + spilled, + ), + ], + ), + one( + NEUTRAL_SCOPE, + case( + "oversized", + "oversized/long_key", + "a 4 KiB key inside a registry prefix family", + "op", + vec![s(&big_key, "v")], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "oversized", + "oversized/deep_array", + "a deeply nested array value", + "op", + vec![array( + "llm.messages", + vec![array( + "", + vec![array("", vec![array("", vec![s("", "deep")])])], + )], + )], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "oversized", + "oversized/long_span_name", + "a 4 KiB span name", + &"n".repeat(4096), + vec![s("gen_ai.operation.name", "chat")], + ), + ), + ] +} + +/// Pseudo-keys are real columns on both sides. The interesting cases are span +/// attributes that impersonate one, and the `value_prefix` op (D1), which the current +/// registry never uses — covered here at the canonicalization level so the columns it +/// would read are exercised. +fn pseudo_key_groups() -> Vec { + vec![ + one( + "openinference.instrumentation.agno", + case( + "pseudo_keys", + "pseudo/span_attribute_named_scope_name", + "a span attribute literally called scope.name must not shadow the column", + "Agent.run", + vec![ + s("scope.name", "litellm"), + s("openinference.span.kind", "AGENT"), + ], + ), + ), + one( + NEUTRAL_SCOPE, + case( + "pseudo_keys", + "pseudo/span_attribute_named_span_name", + "a span attribute called span.name against effect_ai's span-name matchers", + "op", + vec![ + s("span.name", "LanguageModel.generateText"), + s("telemetry.sdk.name", "@effect/opentelemetry"), + ], + ), + ), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "pseudo_keys", + "pseudo/effect_ai_span_name_match", + "the real span-name matcher, promoted by the resource candidate", + "Chat.generateText", + vec![s("http.route", "/x")], + )], + ) + .resource(vec![ + s("service.name", "effect-app"), + s("telemetry.sdk.name", "@effect/opentelemetry"), + ]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "pseudo_keys", + "pseudo/scope_version_and_schema_url", + "scope.version / scope.schema_url carry values (no matcher reads them today)", + "op", + vec![s("gen_ai.operation.name", "chat")], + )], + ) + .scope_version("2.0.0-rc.1+build.7") + .schema_url("https://opentelemetry.io/schemas/1.34.0"), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "pseudo_keys", + "pseudo/empty_scope_version", + "empty scope version and schema url", + "op", + vec![s("openinference.span.kind", "LLM")], + )], + ) + .scope_version(""), + one( + "gen_ai", + case( + "pseudo_keys", + "pseudo/scope_name_eq_is_case_sensitive", + "vercel's insufficient scope matcher, exact match", + "ai.generateText", + vec![s("ai.operationId", "ai.generateText")], + ), + ), + ] +} + +/// Cross-class placement probes: registry keys deliberately put where their matcher's +/// declared class does not look. +/// +/// Both engines are class-directed: a matcher's keys resolve in the one attribute list +/// its class names, and the class-less predicates (unknown-tier fingerprints, session +/// candidates, authority predicates) are span-local. So every span here must classify +/// as if the misplaced key were not there at all. +/// +/// These are the spans that caught the divergence when the Rust side still fell back +/// span → scope → resource for every key: ten of them, plus +/// `not_promoted/langchain_resource` and `typed/bool_false_resource`, were pinned +/// mismatches until 2026-08. They stay in the fixture as the regression surface. +fn cross_class_groups() -> Vec { + vec![ + Group::new( + NEUTRAL_SCOPE, + vec![case( + "cross_class", + "cross_class/resource_key_on_span", + "a resource-class matcher's key carried as a span attribute", + "agent.generate", + vec![s("telemetry.sdk.name", "@mastra/otel-exporter")], + )], + ) + .resource(vec![s("service.name", "app")]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "cross_class", + "cross_class/resource_key_shadowed_by_span", + "the resource carries the matching value; a span attribute of the same name disagrees", + "agent.generate", + vec![s("telemetry.sdk.name", "@opentelemetry/sdk-node")], + )], + ) + .resource(vec![ + s("service.name", "app"), + s("telemetry.sdk.name", "@mastra/otel-exporter"), + ]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "cross_class", + "cross_class/attr_prefix_on_resource", + "an attr-class key_prefix family carried on the resource", + "op", + vec![s("http.route", "/x")], + )], + ) + .resource(vec![s("service.name", "app"), s("agno.run.id", "r-1")]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "cross_class", + "cross_class/attr_key_on_scope", + "an attr-class eq() key carried on the scope", + "op", + vec![s("http.route", "/x")], + )], + ) + .scope_attrs(vec![s("gen_ai.system", "spring_ai")]), + Group::new( + "com.anthropic.claude_code", + vec![case( + "cross_class", + "cross_class/session_key_on_resource", + "the session-candidate key carried on the resource instead of the span", + "claude_code.interaction", + vec![s("span.type", "interaction")], + )], + ) + .resource(vec![s("service.name", "app"), s("session.id", "sess-res")]), + Group::new( + "com.anthropic.claude_code", + vec![case( + "cross_class", + "cross_class/session_key_on_scope", + "the session-candidate key carried on the scope", + "claude_code.interaction", + vec![s("span.type", "interaction")], + )], + ) + .scope_attrs(vec![s("session.id", "sess-scope")]), + Group::new( + "com.anthropic.claude_code", + vec![case( + "cross_class", + "cross_class/authority_key_on_resource", + "the authority predicate's key carried on the resource", + "claude_code.interaction", + vec![s("session.id", "sess-auth-res")], + )], + ) + .resource(vec![s("service.name", "app"), s("span.type", "interaction")]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "cross_class", + "cross_class/unknown_fingerprint_on_resource", + "an unknown-tier fingerprint key carried on the resource", + "op", + vec![s("http.route", "/x")], + )], + ) + .resource(vec![ + s("service.name", "app"), + s("gen_ai.operation.name", "chat"), + ]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "cross_class", + "cross_class/unknown_fingerprint_on_scope", + "an unknown-tier fingerprint key carried on the scope", + "op", + vec![s("http.route", "/x")], + )], + ) + .scope_attrs(vec![s("openinference.span.kind", "LLM")]), + Group::new( + NEUTRAL_SCOPE, + vec![case( + "cross_class", + "cross_class/scope_class_key_on_span", + "a scope-class matcher keyed on a real attribute, carried on the span", + "op", + vec![s("http.route", "/x")], + )], + ) + .resource(vec![ + s("service.name", "gateway"), + s("model_id", "gpt-4o-mini"), + ]), + ] +} + +// --------------------------------------------------------------------------- +// generation +// --------------------------------------------------------------------------- + +fn hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push_str(&format!("{byte:02x}")); + } + out +} + +fn id_bytes(counter: u64, len: usize) -> Vec { + let mut out = vec![0u8; len]; + let source = counter.to_be_bytes(); + for (index, slot) in out.iter_mut().enumerate() { + // Deterministic, never all-zero (bytes_hex() renders an all-zero id as ""). + *slot = source[index % source.len()] ^ (0x5a + index as u8); + } + out +} + +/// Runs every group through the real row writer and returns the JSONL body. +fn generate(groups: &[Group]) -> String { + let datasources = DatasourceNames::defaults(); + let mut seen: BTreeSet = BTreeSet::new(); + let mut out = String::new(); + + for (position, group) in groups.iter().enumerate() { + // 1-based, and the only source of trace/span ids: stable ids are what makes the + // artifact byte-reproducible and what the differential joins fixtures back on. + let counter = position as u64 + 1; + let spans: Vec = group + .cases + .iter() + .enumerate() + .map(|(index, case)| Span { + trace_id: id_bytes(counter, 16), + span_id: id_bytes(counter * 1000 + index as u64 + 1, 8), + parent_span_id: Vec::new(), + trace_state: String::new(), + name: case.span_name.clone(), + kind: span::SpanKind::Internal as i32, + start_time_unix_nano: START_NANOS, + end_time_unix_nano: START_NANOS + 1_000_000, + attributes: case.attributes.clone(), + ..Default::default() + }) + .collect(); + + let request = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: group.resource.clone(), + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }), + scope_spans: vec![ScopeSpans { + scope: group.scope.clone(), + spans, + schema_url: group.schema_url.clone(), + }], + schema_url: String::new(), + }], + }; + + let settings = + AiClassificationSettings::at(true, RECEIVE_SECS); + let (frames, stats) = encode_traces( + &datasources, + ORG_ID, + &request, + &SamplingPolicy::default(), + &[], + &settings, + ) + .expect("encode_traces"); + assert_eq!( + stats.rows, + group.cases.len(), + "every span must produce a row" + ); + assert_eq!(stats.ai_spans_examined, stats.rows); + + let payload = String::from_utf8(frames[0].payload.clone()).expect("utf8 rows"); + let rows: Vec<&str> = payload.lines().filter(|line| !line.is_empty()).collect(); + assert_eq!(rows.len(), group.cases.len()); + + // The classifier, called directly, so the raw session-key value (never stored) + // is available for the hash-alignment leg. + let resource_context = ResourceContext::new(registry(), &group.resource); + let scope_context = resource_context.scope(group.scope.as_ref(), &group.schema_url); + + for (case, row_text) in group.cases.iter().zip(rows) { + assert!( + seen.insert(case.id.clone()), + "duplicate fixture id {}", + case.id + ); + let row: Value = serde_json::from_str(row_text).expect("row json"); + let classification = scope_context.classify_span(&case.span_name, &case.attributes); + let vendor = classification.vendor_slug().to_string(); + let state = classification.session_state; + let hash = classification.session_key_hash(); + + // The direct call and the row writer must agree — otherwise the fixture's + // `rust` block would describe a classification the row never carried. + assert_eq!( + row["ai_vendor"].as_str(), + Some(vendor.as_str()), + "{}", + case.id + ); + assert_eq!( + row["ai_session_key_state"].as_u64(), + Some(state as u64), + "{}", + case.id + ); + assert_eq!( + row["ai_session_key_hash"].as_u64(), + Some(hash), + "{}", + case.id + ); + assert_eq!( + row["ai_rules_version"].as_u64(), + Some(registry().version() as u64), + "{}", + case.id + ); + + let session_key = classification.session_key.as_deref(); + if let Some(value) = session_key { + // Third leg of the hash claim: the classifier's own hash is exactly + // cityhash102 over the pinned construction. + assert_eq!( + hash, + crate::cityhash102::city_hash64(value.as_bytes()), + "{}", + case.id + ); + } + + let record = json!({ + "id": case.id, + "category": case.category, + "note": case.note, + "rust": { + "vendor": vendor, + "session_state": state, + // A string: u64 hashes above 2^53 do not survive JSON.parse. + "session_key_hash": hash.to_string(), + "rules_version": registry().version(), + // Hex of the raw winning value, which no row carries. The + // hash-contract e2e replays exactly these bytes into ClickHouse. + "session_key_hex": session_key.map(|value| hex(value.as_bytes())), + }, + }); + out.push_str(&serde_json::to_string(&record).expect("record json")); + out.push('\n'); + } + } + out +} + +fn fixture_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/adversarial/adversarial-spans.jsonl") +} + +// --------------------------------------------------------------------------- +// tests +// --------------------------------------------------------------------------- + +/// Writes the artifact. Opt-in, because a test that writes into the source tree on +/// every `cargo test` would fight the determinism check it exists to feed. +#[test] +#[ignore = "regeneration: set ADVERSARIAL_FIXTURE_OUT"] +fn write_adversarial_fixture() { + let Some(out) = std::env::var_os("ADVERSARIAL_FIXTURE_OUT") else { + panic!("set ADVERSARIAL_FIXTURE_OUT to the artifact path"); + }; + let body = generate(&groups()); + let path = PathBuf::from(out); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).expect("create fixture directory"); + } + std::fs::write(&path, &body).expect("write fixture"); + println!( + "wrote {} spans, {} bytes to {}", + body.lines().count(), + body.len(), + path.display() + ); +} + +/// Regeneration is byte-stable: a missing artifact fails, a stale one fails with the +/// moved line. +/// +/// This used to skip when the artifact was absent, so that "a fresh checkout of +/// apps/ingest alone still builds". That rationale is dead — since the TS-mirror test +/// the crate `include_str!`s `packages/domain/src/ai/vendors.ts` under `cfg(test)`, so +/// an apps/ingest-only tree does not compile at all. Meanwhile this golden is the only +/// gate on several rules (tier-2's guard, session authority for non-entry vendors), so +/// a sparse checkout or a bad merge that dropped the file would have left CI green with +/// the gate gone. +#[test] +fn fixture_is_reproducible() { + let path = fixture_path(); + let checked_in = std::fs::read_to_string(&path).unwrap_or_else(|error| { + panic!( + "{} is missing ({error}) — this golden is the only gate on several rules; \ + regenerate it with ADVERSARIAL_FIXTURE_OUT (see this module's header)", + path.display() + ) + }); + let regenerated = generate(&groups()); + if regenerated != checked_in { + let expected: Vec<&str> = checked_in.lines().collect(); + let actual: Vec<&str> = regenerated.lines().collect(); + let first_difference = expected + .iter() + .zip(&actual) + .position(|(left, right)| left != right); + panic!( + "{} is stale ({} lines checked in, {} regenerated, first differing line {:?}). \ + Regenerate with ADVERSARIAL_FIXTURE_OUT= cargo test --lib write_adversarial_fixture -- --ignored", + path.display(), + expected.len(), + actual.len(), + first_difference + ); + } +} + +/// Composition guard: every branch the plan's §6 fuzz surface names must be present, +/// so a future edit cannot quietly shrink the corpus. +#[test] +fn fixture_covers_every_branch() { + let body = generate(&groups()); + let mut categories: BTreeMap = BTreeMap::new(); + let mut vendors: BTreeSet = BTreeSet::new(); + let mut states: BTreeSet = BTreeSet::new(); + for line in body.lines() { + let record: Value = serde_json::from_str(line).expect("record"); + *categories + .entry(record["category"].as_str().expect("category").to_string()) + .or_default() += 1; + vendors.insert( + record["rust"]["vendor"] + .as_str() + .expect("vendor") + .to_string(), + ); + states.insert(record["rust"]["session_state"].as_u64().expect("state")); + } + + for category in [ + "resolution/sufficient_scope", + "resolution/insufficient_promoted", + "resolution/insufficient_not_promoted", + "resolution/attr_only", + "resolution/cross_vendor", + "resolution/unknown_tier", + "resolution/non_ai", + "values/typed", + "values/present_empty", + "keys/duplicate", + "keys/near_miss", + "unicode", + "oversized", + "pseudo_keys", + "cross_class", + ] { + assert!( + categories.contains_key(category), + "missing category {category}" + ); + } + // Every session-key state, 0..=6. + assert_eq!( + states, + (0..=6u64).collect::>(), + "states covered" + ); + // Every vendor in the registry classifies at least one fixture span. + for vendor in registry().vendors() { + assert!( + vendors.contains(vendor.slug()), + "no fixture span classifies as {}", + vendor.slug() + ); + } + assert!( + body.lines().count() >= 250, + "corpus is too small to be adversarial" + ); + assert!(body.len() < 1_000_000, "artifact must stay under ~1 MB"); +} diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 200013018..7d9ba63bc 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,12 +1,12 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7"; +pub const PROJECT_REVISION: &str = "3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse // clickHouseSchemaVersion. -pub const SCHEMA_VERSION: &str = "13"; +pub const SCHEMA_VERSION: &str = "16"; pub const ORG_PLACEHOLDER: &str = "__ORG__"; #[derive(Debug)] @@ -22,9 +22,9 @@ pub const DATASOURCES: &[InsertMapping] = &[ InsertMapping { datasource: "traces", table: "traces", - columns: &["OrgId", "Timestamp", "TraceId", "SpanId", "ParentSpanId", "TraceState", "SpanName", "SpanKind", "ServiceName", "ResourceSchemaUrl", "ResourceAttributes", "ScopeSchemaUrl", "ScopeName", "ScopeVersion", "ScopeAttributes", "Duration", "StatusCode", "StatusMessage", "SpanAttributes", "EventsTimestamp", "EventsName", "EventsAttributes", "LinksTraceId", "LinksSpanId", "LinksTraceState", "LinksAttributes"], - selects: &["__ORG__", "start_time", "trace_id", "span_id", "parent_span_id", "trace_state", "span_name", "span_kind", "service_name", "resource_schema_url", "resource_attributes", "scope_schema_url", "scope_name", "scope_version", "scope_attributes", "duration", "status_code", "status_message", "span_attributes", "events_timestamp", "events_name", "events_attributes", "links_trace_id", "links_span_id", "links_trace_state", "links_attributes"], - input_schema: "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))", + columns: &["OrgId", "Timestamp", "TraceId", "SpanId", "ParentSpanId", "TraceState", "SpanName", "SpanKind", "ServiceName", "ResourceSchemaUrl", "ResourceAttributes", "ScopeSchemaUrl", "ScopeName", "ScopeVersion", "ScopeAttributes", "Duration", "StatusCode", "StatusMessage", "SpanAttributes", "EventsTimestamp", "EventsName", "EventsAttributes", "LinksTraceId", "LinksSpanId", "LinksTraceState", "LinksAttributes", "AiVendor", "AiSessionKeyState", "AiSessionKeyHash", "AiRulesVersion", "AiRollupHour"], + selects: &["__ORG__", "start_time", "trace_id", "span_id", "parent_span_id", "trace_state", "span_name", "span_kind", "service_name", "resource_schema_url", "resource_attributes", "scope_schema_url", "scope_name", "scope_version", "scope_attributes", "duration", "status_code", "status_message", "span_attributes", "events_timestamp", "events_name", "events_attributes", "links_trace_id", "links_span_id", "links_trace_state", "links_attributes", "ai_vendor", "ai_session_key_state", "ai_session_key_hash", "ai_rules_version", "ai_rollup_hour"], + input_schema: "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String)), ai_vendor LowCardinality(String), ai_session_key_state UInt8, ai_session_key_hash UInt64, ai_rules_version UInt32, ai_rollup_hour DateTime('UTC')", }, InsertMapping { datasource: "logs", diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index 8e84b3790..6a8ee206f 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -43,9 +43,10 @@ use maple_ingest::session_analytics::{ derive_referrer_host, sanitize_session_event, sanitize_session_meta, }; use maple_ingest::telemetry::{ - AttributeMappingRule, ClickHouseBreakerConfig, ClickHouseTarget, ClickHouseTargetProvider, - DatasourceNames, ExportDestination, MappingOperation, MappingSourceContext, PipelineError, - SamplingPolicy, TelemetryPipeline, TelemetrySignal, TinybirdConfig, + AiClassificationSettings, AttributeMappingRule, ClickHouseBreakerConfig, ClickHouseTarget, + ClickHouseTargetProvider, DatasourceNames, ExportDestination, MappingOperation, + MappingSourceContext, PipelineError, SamplingPolicy, TelemetryPipeline, TelemetrySignal, + TinybirdConfig, }; use maple_ingest::usage_metrics::{billable_gb, usage_cardinality_view, UsageMetrics}; use moka::future::Cache; @@ -122,6 +123,12 @@ struct AppConfig { max_request_body_bytes: usize, org_max_in_flight: u64, require_tls: bool, + /// `INGEST_AI_CLASSIFICATION_ENABLED`. **Migration-window flag**, default + /// false: it exists only to ramp the write-side plan's §7 step 2 shadow + /// deploy and is removed once classification is unconditional in production + /// (§7 step 4). Read once per batch, never per span. `AiRollupHour` is + /// written on every row regardless of this flag. + ai_classification_enabled: bool, key_store_backend: KeyStoreBackend, clickhouse_encryption_key: Option<[u8; 32]>, lookup_hmac_key: String, @@ -349,8 +356,8 @@ impl AppConfig { let key_store_backend = resolve_key_store_backend()?; let clickhouse_encryption_key = match &key_store_backend { - // The Postgres key store decrypts BYO-ClickHouse credentials from - // org_clickhouse_settings, so it needs the encryption key. + // MAPLE_INGEST_KEY_ENCRYPTION_KEY, for BYO-ClickHouse credentials + // from org_clickhouse_settings, encrypted by apps/api with the same key. KeyStoreBackend::Postgres { .. } => { let raw = std::env::var("MAPLE_INGEST_KEY_ENCRYPTION_KEY") .map_err(|_| "MAPLE_INGEST_KEY_ENCRYPTION_KEY is required".to_string())?; @@ -460,6 +467,14 @@ impl AppConfig { } }; + // Default off: the flag ramps classification per-deployment during the + // migration window and is deleted afterwards. + let ai_classification_enabled = parse_bool( + "INGEST_AI_CLASSIFICATION_ENABLED", + std::env::var("INGEST_AI_CLASSIFICATION_ENABLED").ok(), + false, + )?; + // Default off — see the field doc. Set it on services that are only // reachable through Cloudflare. let trust_proxy_geo = parse_bool( @@ -478,6 +493,7 @@ impl AppConfig { max_request_body_bytes, org_max_in_flight, require_tls, + ai_classification_enabled, key_store_backend, clickhouse_encryption_key, lookup_hmac_key, @@ -1513,14 +1529,14 @@ async fn main() { // process exit, so a bad deploy never goes ready while a transient database // fault can no longer kill a healthy running fleet. let key_store_ready = Arc::new(AtomicBool::new(false)); - let store: Arc = match build_key_store(&config, Arc::clone(&key_store_ready)).await - { - Ok(store) => store, - Err(error) => { - eprintln!("Key store init error: {error}"); - std::process::exit(1); - } - }; + let store: Arc = + match build_key_store(&config, Arc::clone(&key_store_ready)).await { + Ok(store) => store, + Err(error) => { + eprintln!("Key store init error: {error}"); + std::process::exit(1); + } + }; // The Postgres key store resolves BYO-ClickHouse export targets from // org_clickhouse_settings (the Static backend has no DB to resolve from). @@ -2050,6 +2066,7 @@ async fn resolve_grpc_ingest_key( key_id: "sentinel".to_string(), self_managed: false, clickhouse_ready: false, + // Health-probe traffic is discarded, never classified. }); } @@ -2079,7 +2096,11 @@ async fn ready(State(state): State>) -> Response { if state.key_store_ready.load(Ordering::Relaxed) { (StatusCode::OK, "READY").into_response() } else { - (StatusCode::SERVICE_UNAVAILABLE, "DEGRADED: key store unavailable").into_response() + ( + StatusCode::SERVICE_UNAVAILABLE, + "DEGRADED: key store unavailable", + ) + .into_response() } } @@ -3410,7 +3431,9 @@ async fn handle_cloudflare_logpush_inner( key_id: resolved.secret_key_id.clone(), self_managed: resolved.self_managed, clickhouse_ready: resolved.clickhouse_ready, - }; + // Cloudflare Logpush is a logs-only connector; logs + // classification is v2 (write-side plan §3). + }; let decoded = DecodedPayload::Logs(request); let reservation = reserve_autumn_usage( state, @@ -4241,12 +4264,17 @@ async fn accept_native_decoded_payload( "maple.ingest.attribute_mapping_count", attribute_mappings.len(), ); + // The flag is already resolved from the env; this reads it once for + // the whole batch. + let ai = AiClassificationSettings::new(state.config.ai_classification_enabled); + Span::current().record("maple.ingest.ai.enabled", ai.enabled); pipeline .accept_traces_to( &resolved_key.org_id, request, &policy, attribute_mappings.as_slice(), + &ai, destination, ) .await @@ -4283,6 +4311,13 @@ async fn accept_native_decoded_payload( if stats.dropped > 0 { metrics::native_sampled_dropped(signal.path(), stats.dropped as u64); } + if stats.ai_spans_examined > 0 { + metrics::ai_spans_examined(signal.path(), stats.ai_spans_examined as u64); + Span::current().record( + "maple.ingest.ai.spans_examined", + stats.ai_spans_examined as u64, + ); + } Span::current().record("maple.ingest.native_rows", stats.rows as u64); Span::current().record("maple.ingest.sampled_dropped", stats.dropped as u64); Ok(()) @@ -4758,7 +4793,12 @@ impl PostgresKeyStore { WHERE k.private_key_hash = $1 LIMIT 1", &[&"__ingest_probe_no_match__"], ) - .instrument(postgres_client_span("probe", "SELECT", "org_ingest_keys", &self.target)) + .instrument(postgres_client_span( + "probe", + "SELECT", + "org_ingest_keys", + &self.target, + )) .await .map(|_| ()) .map_err(|error| format!("postgres probe query failed: {}", error_chain(&error))) @@ -4793,7 +4833,9 @@ impl KeyStore for PostgresKeyStore { &self.target, )) .await - .map_err(|error| format!("postgres fetch_ingest_key failed: {}", error_chain(&error)))?; + .map_err(|error| { + format!("postgres fetch_ingest_key failed: {}", error_chain(&error)) + })?; let Some(row) = rows.into_iter().next() else { return Ok(None); }; @@ -6192,6 +6234,51 @@ mod tests { } } + /// One Spring AI span, mirroring `telemetry.rs`'s AI fixture: an app-chosen + /// (insufficient) scope promoted by a `spring.ai.` attribute hit, with + /// spring_ai's session-granularity key present. + fn test_ai_trace_request() -> ExportTraceServiceRequest { + use opentelemetry_proto::tonic::trace::v1::{span, ResourceSpans, ScopeSpans, Span}; + let string_kv = |key: &str, value: &str| KeyValue { + key: key.to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(value.to_string())), + }), + }; + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![string_kv("service.name", "spring-ai-app")], + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: "org.springframework.boot".to_string(), + version: "4.1.0".to_string(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + spans: vec![Span { + trace_id: vec![0x77; 16], + span_id: vec![0x88; 8], + name: "chat_client".to_string(), + kind: span::SpanKind::Internal as i32, + start_time_unix_nano: 1_700_000_000_000_000_000, + end_time_unix_nano: 1_700_000_000_500_000_000, + attributes: vec![ + string_kv("spring.ai.kind", "chat_client"), + string_kv("spring.ai.chat.client.conversation.id", "sess-e2e"), + ], + ..Default::default() + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + } + fn test_headers(raw_key: &str) -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert( @@ -6209,6 +6296,18 @@ mod tests { queue_dir: PathBuf, forward_endpoint: String, routing_ttl: Duration, + ) -> AppState { + test_app_state_with_ai(store, queue_dir, forward_endpoint, routing_ttl, false).await + } + + /// `test_app_state` with `INGEST_AI_CLASSIFICATION_ENABLED` settable, so the + /// flag's real path through `AppConfig` can be exercised. + async fn test_app_state_with_ai( + store: Arc, + queue_dir: PathBuf, + forward_endpoint: String, + routing_ttl: Duration, + ai_classification_enabled: bool, ) -> AppState { let tinybird = test_tinybird_config(queue_dir); let key_store: Arc = store.clone(); @@ -6246,6 +6345,7 @@ mod tests { max_request_body_bytes: 1024 * 1024, org_max_in_flight: 100, require_tls: false, + ai_classification_enabled, key_store_backend: KeyStoreBackend::Static { org_id: "org_test".to_string(), }, @@ -6441,11 +6541,7 @@ mod tests { walk(dir) } - async fn replay_blob_test_state( - raw_key: &str, - org_id: &str, - queue_dir: PathBuf, - ) -> AppState { + async fn replay_blob_test_state(raw_key: &str, org_id: &str, queue_dir: PathBuf) -> AppState { let store = Arc::new(FakeKeyStore::default()); store.insert_private( raw_key, @@ -6515,9 +6611,9 @@ mod tests { assert_eq!(captured.content_type, "application/json"); assert_eq!(captured.content_encoding, "gzip"); assert!( - captured.authorization.starts_with( - "AWS4-HMAC-SHA256 Credential=test-access-key/" - ), + captured + .authorization + .starts_with("AWS4-HMAC-SHA256 Credential=test-access-key/"), "expected a SigV4 authorization header, got {:?}", captured.authorization ); @@ -7213,6 +7309,99 @@ mod tests { assert_eq!(plaintext, "ch-secret-123"); } + /// The whole write path with the flag on: HTTP request → key resolve → + /// `AppConfig.ai_classification_enabled` → classification → the NDJSON body + /// ClickHouse actually receives. + #[tokio::test] + async fn ai_classification_flag_reaches_the_clickhouse_row() { + let (ch_tx, mut ch_rx) = tokio::sync::mpsc::unbounded_channel(); + let ch_listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let ch_addr = ch_listener.local_addr().unwrap(); + let ch_app = Router::new() + .route("/", post(fake_clickhouse_import)) + .with_state(ch_tx); + tokio::spawn(async move { + axum::serve(ch_listener, ch_app).await.unwrap(); + }); + + let queue_dir = unique_main_test_dir("ai-classification"); + let store = Arc::new(FakeKeyStore::default()); + let raw_key = "maple_sk_test_ai_flag"; + store.insert_private( + raw_key, + KeyRow { + org_id: "org_ai_flag".to_string(), + self_managed: true, + clickhouse_ready: true, + }, + ); + store.insert_clickhouse_target( + "org_ai_flag", + ClickHouseTargetRow { + ch_url: format!("http://{ch_addr}"), + ch_user: "ingest".to_string(), + ch_password_ciphertext: None, + ch_password_iv: None, + ch_password_tag: None, + ch_database: "maple".to_string(), + schema_version: CLICKHOUSE_SCHEMA_VERSION.to_string(), + }, + ); + let state = test_app_state_with_ai( + Arc::clone(&store), + queue_dir.clone(), + "http://127.0.0.1:1".to_string(), + Duration::from_millis(5), + true, + ) + .await; + + let (response, item_count, _, _, _) = handle_signal_inner( + &state, + &test_headers(raw_key), + Bytes::from(test_ai_trace_request().encode_to_vec()), + Signal::Traces, + ) + .await + .expect("request should be accepted through the native ClickHouse path"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(item_count, 1); + + let import = tokio::time::timeout(Duration::from_secs(2), ch_rx.recv()) + .await + .expect("ready org should write to ClickHouse") + .expect("ClickHouse channel should stay open"); + + // The INSERT must name the five columns, or the values below would land + // in the wrong ones. + assert!(import.query.contains("AiVendor"), "{}", import.query); + assert!(import.query.contains("AiRollupHour"), "{}", import.query); + + let row: serde_json::Value = serde_json::from_str(import.body.trim()).expect("NDJSON row"); + assert_eq!(row["ai_vendor"], "spring_ai"); + assert_eq!(row["ai_session_key_state"], serde_json::json!(6)); + assert_eq!( + row["ai_session_key_hash"], + serde_json::json!(maple_ingest::cityhash102::city_hash64(b"sess-e2e")) + ); + assert_ne!(row["ai_rules_version"], serde_json::json!(0)); + // The fixture's span start is a fixed 2023 timestamp, i.e. far outside + // the [receive − 7d, receive + 1d] window, so the clamp must relocate it + // to the hour this batch was received rather than minting a 2023 + // partition. (The clamp's window arithmetic is covered exhaustively in + // `telemetry.rs` against an injected receive time.) + let now_secs = (current_time_millis() / 1000) as i64; + let this_hour = chrono::DateTime::from_timestamp(now_secs - now_secs.rem_euclid(3600), 0) + .expect("valid receive hour") + .format("%Y-%m-%d %H:%M:%S") + .to_string(); + assert_eq!(row["ai_rollup_hour"], serde_json::json!(this_hour)); + + let _ = std::fs::remove_dir_all(queue_dir); + } + #[test] fn schema_revision_accepts_newer_schemas_and_rejects_older_ones() { let needed = required_schema_revision(); diff --git a/apps/ingest/src/metrics.rs b/apps/ingest/src/metrics.rs index dbd98c078..c2ccc2546 100644 --- a/apps/ingest/src/metrics.rs +++ b/apps/ingest/src/metrics.rs @@ -140,6 +140,13 @@ static NATIVE_ROWS_TOTAL: LazyLock> = LazyLock::new(|| { .build() }); +static AI_SPANS_EXAMINED_TOTAL: LazyLock> = LazyLock::new(|| { + METER + .u64_counter("ingest_ai_spans_examined_total") + .with_description("Spans the AI classifier examined, by signal") + .build() +}); + static NATIVE_SAMPLED_DROPPED_TOTAL: LazyLock> = LazyLock::new(|| { METER .u64_counter("ingest_native_sampled_dropped_total") @@ -583,6 +590,18 @@ pub fn native_rows(signal: &str, count: u64) { NATIVE_ROWS_TOTAL.add(count, &[KeyValue::new("signal", signal.to_string())]); } +/// Spans the AI classifier examined — write-side plan §4's completeness signal. +/// +/// Labeled by `signal` only, exactly like `native_rows`, so the two are directly +/// comparable: with classification enabled every accepted trace row is an +/// examined span, and any divergence between these series is a bug (a code path +/// building rows without classifying, or a partially-flagged fleet). It is +/// deliberately *not* org-labeled — this counter moves constantly on a healthy +/// fleet, which is the opposite of `org_data_loss`'s cardinality budget. +pub fn ai_spans_examined(signal: &str, count: u64) { + AI_SPANS_EXAMINED_TOTAL.add(count, &[KeyValue::new("signal", signal.to_string())]); +} + /// Rows dropped by sampling in the native pipeline. pub fn native_sampled_dropped(signal: &str, count: u64) { NATIVE_SAMPLED_DROPPED_TOTAL.add(count, &[KeyValue::new("signal", signal.to_string())]); diff --git a/apps/ingest/src/otel.rs b/apps/ingest/src/otel.rs index de50e880c..7902bd52d 100644 --- a/apps/ingest/src/otel.rs +++ b/apps/ingest/src/otel.rs @@ -382,6 +382,12 @@ pub fn accept_internal_span(signal_path: &'static str, destination: &'static str "maple.ingest.destination" = destination, "maple.ingest.native_rows" = Empty, "maple.ingest.sampled_dropped" = Empty, + // Whether the migration-window classification flag was on for this + // batch, and how many spans it examined. Batch-level, not per span: + // per-span classification spans on this hot path are exactly what + // CLAUDE.md's self-observability rule forbids. + "maple.ingest.ai.enabled" = Empty, + "maple.ingest.ai.spans_examined" = Empty, ) } diff --git a/apps/ingest/src/telemetry.rs b/apps/ingest/src/telemetry.rs index a3b765d85..815499598 100644 --- a/apps/ingest/src/telemetry.rs +++ b/apps/ingest/src/telemetry.rs @@ -7,6 +7,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; +use crate::ai_classifier::{self, ResourceContext, SpanClassification}; +use crate::ai_registry::registry; use crate::clickhouse_insert_mappings::{self, InsertMapping}; use crate::metrics; use crate::otel::{ @@ -602,6 +604,132 @@ impl PipelineError { pub struct AcceptStats { pub rows: usize, pub dropped: usize, + /// Spans the AI classifier examined in this batch. Zero when the migration + /// flag is off; otherwise equal to `rows` for traces (plan §4's + /// `spans_ingested` vs `spans_examined` completeness pair). + pub ai_spans_examined: usize, +} + +/// Per-batch inputs for AI classification and rollup-hour clamping. +/// +/// Built once per accepted payload, never per span: the flag is read once and +/// `receive_time_secs` is captured a single time so every span in one payload +/// clamps against the same instant. +#[derive(Clone, Debug)] +pub struct AiClassificationSettings { + /// `INGEST_AI_CLASSIFICATION_ENABLED`. **Migration-window flag only** — the + /// write-side plan (§7 step 4) ramps it to 100% across the fleet, records that + /// hour as `AI_VENDORS_ROLLUP_ENABLEMENT_HOUR`, and then removes the flag; + /// production classifies unconditionally. There is no full-clock-hour + /// condition and no ordering against MV creation — the MV ships with the + /// migration chain and is a no-op until the ramp. `AiRollupHour` is written + /// whether this is set or not. + pub enabled: bool, + /// Batch receive time, epoch seconds. + pub receive_time_secs: i64, +} + +impl AiClassificationSettings { + pub fn new(enabled: bool) -> Self { + Self { + enabled, + receive_time_secs: unix_now_secs(), + } + } + + /// Flag off. `AiRollupHour` is still computed and written. + pub fn disabled() -> Self { + Self::new(false) + } + + /// Explicit receive time, for tests that assert the clamp windows. + #[cfg(test)] + pub fn at(enabled: bool, receive_time_secs: i64) -> Self { + Self { + enabled, + receive_time_secs, + } + } +} + +fn unix_now_secs() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs() as i64) + .unwrap_or(0) +} + +/// The four classification columns as the row writes them. Extracted from a +/// [`SpanClassification`] immediately so the row builder never has to hold a +/// borrow of the span's attribute list. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct AiRowFields { + vendor: &'static str, + session_state: u8, + session_key_hash: u64, + rules_version: u32, +} + +impl AiRowFields { + /// Flag-off values. `rules_version = 0` is the plan's pre-rollout marker: + /// `AiRulesVersion = 0` means "never examined", which is what a flag-off row + /// is, and is distinguishable from an examined-and-non-AI row (non-zero + /// version, empty vendor). + const UNEXAMINED: Self = Self { + vendor: "", + session_state: ai_classifier::session_state::NOT_EXAMINED, + session_key_hash: 0, + rules_version: 0, + }; + + fn from_classification(classification: &SpanClassification<'_>) -> Self { + Self { + vendor: classification.vendor_slug(), + session_state: classification.session_state, + session_key_hash: classification.session_key_hash(), + rules_version: classification.rules_version, + } + } +} + +/// Seconds in the clamp window's past and future halves (write-side plan §3). +const ROLLUP_CLAMP_PAST_SECS: i64 = 7 * 24 * 60 * 60; +const ROLLUP_CLAMP_FUTURE_SECS: i64 = 24 * 60 * 60; + +/// `AiRollupHour`: `toStartOfHour(Timestamp)` when the span's start time is +/// within `[receive − 7d, receive + 1d]`, else `toStartOfHour(receive_time)`. +/// +/// Client timestamps are attacker- and replay-controlled, and this column is the +/// rollup's partition key, so an unclamped value means unbounded partition +/// creation and rows whose TTL never fires. Clamping at write time rather than +/// in the MV is the only deterministic option: an MV-side clamp would need +/// `now()`, which a later partition rebuild re-evaluates at rebuild time and +/// would silently relocate rows across hours. +fn rollup_hour_secs(span_start_unix_nano: u64, receive_time_secs: i64) -> i64 { + let span_secs = (span_start_unix_nano / 1_000_000_000) as i64; + let in_window = span_secs >= receive_time_secs - ROLLUP_CLAMP_PAST_SECS + && span_secs <= receive_time_secs + ROLLUP_CLAMP_FUTURE_SECS; + let chosen = if in_window { + span_secs + } else { + receive_time_secs + }; + chosen - chosen.rem_euclid(3600) +} + +/// `DateTime('UTC')` wire format: `YYYY-MM-DD HH:MM:SS`, the second-precision +/// sibling of `format_timestamp_nano`'s `DateTime64(9)` rendering. +/// +/// A string, not epoch seconds. Both destinations accept either — Tinybird's +/// JSONPath ingestion and ClickHouse's `input('… AiRollupHour DateTime(\'UTC\')')` +/// both parse a quoted datetime — but every other timestamp this row writer +/// emits is a string in this shape, and a lone numeric column would make the +/// JSON row's timestamps inconsistent to read and to diff. +fn format_datetime_secs(unix_secs: i64) -> String { + match chrono::DateTime::from_timestamp(unix_secs, 0) { + Some(dt) => dt.format("%Y-%m-%d %H:%M:%S").to_string(), + None => "1970-01-01 00:00:00".to_string(), + } } #[derive(Clone)] @@ -721,12 +849,14 @@ impl TelemetryPipeline { request: &ExportTraceServiceRequest, sampling_policy: &SamplingPolicy, attribute_mappings: &[AttributeMappingRule], + ai: &AiClassificationSettings, ) -> Result { self.accept_traces_to( org_id, request, sampling_policy, attribute_mappings, + ai, ExportDestination::Tinybird, ) .await @@ -738,6 +868,7 @@ impl TelemetryPipeline { request: &ExportTraceServiceRequest, sampling_policy: &SamplingPolicy, attribute_mappings: &[AttributeMappingRule], + ai: &AiClassificationSettings, destination: ExportDestination, ) -> Result { let (frames, stats) = { @@ -749,6 +880,7 @@ impl TelemetryPipeline { request, sampling_policy, attribute_mappings, + ai, )?; record_encode_stats(&span, &frames, &stats); (frames, stats) @@ -841,6 +973,8 @@ impl TelemetryPipeline { let stats = AcceptStats { rows: rows.len(), dropped: 0, + // Session-replay rows, not spans — nothing to classify. + ai_spans_examined: 0, }; let frames = rows_to_frames(org_id, hash64(org_id), signal, datasource, rows); self.commit_frames(frames, destination).await?; @@ -2183,23 +2317,39 @@ fn encode_traces( request: &ExportTraceServiceRequest, policy: &SamplingPolicy, attribute_mappings: &[AttributeMappingRule], + ai: &AiClassificationSettings, ) -> Result<(Vec, AcceptStats), PipelineError> { let mut rows = Vec::with_capacity(count_trace_rows(request)); let mut dropped = 0usize; + let mut ai_spans_examined = 0usize; let mut routing_key = hash64(org_id); let sample_ratio = policy.clamped_ratio(); let sample_rate = 1.0 / sample_ratio; + // Rules can add, rename or delete registry-referenced span attributes, so a + // span classified from the wire list would not be the span the row stores. + // With no rules configured — the overwhelmingly common case — the Map is a + // faithful canonicalization of the wire list, so the wire list *is* the row. + let remapped = !attribute_mappings.is_empty(); for resource_spans in &request.resource_spans { let resource = resource_spans.resource.as_ref(); - let resource_attrs = resource - .map(|resource| attr_map(&resource.attributes)) - .unwrap_or_default(); + let resource_attributes = resource + .map(|resource| resource.attributes.as_slice()) + .unwrap_or(&[]); + let resource_attrs = attr_map(resource_attributes); let service_name = resource_attrs .get("service.name") .and_then(Value::as_str) .unwrap_or("") .to_string(); + // Hoisted once per ResourceSpans, per the plan's §2 batch algorithm — + // but only when a span will actually use it. With the flag off there is + // nothing to classify, so hoisting would be pure waste on the default + // (flag-off) path. Attribute mappings rewrite *span* attributes only, so + // the resource/scope contexts are valid for the remapped path too. + let ai_resource = ai + .enabled + .then(|| ResourceContext::new(registry(), resource_attributes)); for scope_spans in &resource_spans.scope_spans { let scope = scope_spans.scope.as_ref(); @@ -2208,6 +2358,10 @@ fn encode_traces( .unwrap_or_default(); let scope_name = scope.map(|scope| scope.name.as_str()).unwrap_or(""); let scope_version = scope.map(|scope| scope.version.as_str()).unwrap_or(""); + // Hoisted once per ScopeSpans; every span below reuses it. + let ai_scope = ai_resource + .as_ref() + .map(|resource| resource.scope(scope, &scope_spans.schema_url)); for span in &scope_spans.spans { let trace_id = bytes_hex(&span.trace_id); @@ -2222,16 +2376,64 @@ fn encode_traces( } let mut span_attrs = attr_map(&span.attributes); + // `attr_map` keeps the LAST duplicate (the storage rule); the + // classifier's contract is the FIRST occurrence. Without mapping + // rules the classifier reads the wire list and gets first-wins for + // free — with them it reads the remapped Map, so a span that + // actually carries a duplicate key needs a parallel first-wins view + // or the mere presence of an unrelated rule flips its verdict. The + // length comparison is exact: `attr_map` only ever collapses + // duplicates, so equal lengths mean the two rules agree and the + // common path allocates nothing. + let mut classify_attrs = + (ai.enabled && remapped && span.attributes.len() != span_attrs.len()) + .then(|| attr_map_first_wins(&span.attributes)); if sample_ratio < 1.0 && !span_attrs.contains_key("SampleRate") && !span.trace_state.contains("th:") { - span_attrs.insert( - "SampleRate".to_string(), - json!(format_sample_rate(sample_rate)), - ); + let sample_rate = json!(format_sample_rate(sample_rate)); + if let Some(attrs) = classify_attrs.as_mut() { + attrs.insert("SampleRate".to_string(), sample_rate.clone()); + } + span_attrs.insert("SampleRate".to_string(), sample_rate); } apply_attribute_mappings(attribute_mappings, &resource_attrs, &mut span_attrs); + if let Some(attrs) = classify_attrs.as_mut() { + apply_attribute_mappings(attribute_mappings, &resource_attrs, attrs); + } + + // Classification runs here, after remapping: an org that remaps a + // custom key onto a rule key (a session id, a discriminator) must + // classify by the remapped shape. Flag-off skips the work entirely + // and writes the pre-rollout marker values; `AiRollupHour` below + // is written either way. + let ai_fields = if !ai.enabled { + AiRowFields::UNEXAMINED + } else { + ai_spans_examined += 1; + let scope_context = ai_scope + .as_ref() + .expect("hoisted whenever classification is enabled"); + // The rewritten attribute list is per-span; the hoisted + // context is not — it borrows the resource and scope, which + // mapping rules never touch. Only the list differs. + // `classify_attrs` is the first-wins view built above; it is + // `None` unless this span carries a duplicate key, in which + // case the last-wins Map is already the same list. + let rewritten = remapped.then(|| { + key_values_from_map(classify_attrs.as_ref().unwrap_or(&span_attrs)) + }); + AiRowFields::from_classification(&scope_context.classify_span_full( + &span.name, + rewritten.as_deref().unwrap_or(&span.attributes), + &span.events, + )) + }; + let ai_rollup_hour = format_datetime_secs(rollup_hour_secs( + span.start_time_unix_nano, + ai.receive_time_secs, + )); let events_timestamp: Vec = span .events @@ -2291,7 +2493,12 @@ fn encode_traces( "links_trace_id": links_trace_id, "links_span_id": links_span_id, "links_trace_state": links_trace_state, - "links_attributes": links_attributes + "links_attributes": links_attributes, + "ai_vendor": ai_fields.vendor, + "ai_session_key_state": ai_fields.session_state, + "ai_session_key_hash": ai_fields.session_key_hash, + "ai_rules_version": ai_fields.rules_version, + "ai_rollup_hour": ai_rollup_hour }))?); } } @@ -2300,6 +2507,7 @@ fn encode_traces( let stats = AcceptStats { rows: rows.len(), dropped, + ai_spans_examined, }; let frames = rows_to_frames( org_id, @@ -2360,6 +2568,8 @@ fn encode_logs( let stats = AcceptStats { rows: rows.len(), dropped: 0, + // Logs classification is v2 (write-side plan §3). + ai_spans_examined: 0, }; let frames = rows_to_frames( org_id, @@ -2573,6 +2783,8 @@ fn encode_metrics( AcceptStats { rows: row_count, dropped: 0, + // Metrics classification is v2 (write-side plan §3). + ai_spans_examined: 0, }, )) } @@ -2783,17 +2995,73 @@ fn format_sample_rate(sample_rate: f64) -> String { } } +/// The written span-attribute Map back as an attribute list, so the classifier +/// can be pointed at the post-remapping row. +/// +/// Only built for orgs that actually configured attribute-mapping rules. Values +/// are already canonical strings (they came out of [`any_value_string`]), so +/// re-wrapping them as `StringValue` round-trips exactly. +fn key_values_from_map(attributes: &Map) -> Vec { + attributes + .iter() + .map(|(key, value)| KeyValue { + key: key.clone(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + value.as_str().unwrap_or_default().to_string(), + )), + }), + }) + .collect() +} + +/// Canonicalizes an attribute list into the row's Map column. +/// +/// **Duplicate keys keep the last occurrence** — the historical JSON-object +/// behavior, for every key. The classifier separately dedupes its *own* view of +/// rule-referenced keys first-occurrence-wins (`ai_classifier`); that is a +/// determinism rule for matching, not a storage rule. (v1 coupled the two so SQL +/// over the written row could reproduce the Rust verdict; that contract is gone, +/// and with it a registry probe on the duplicate-key path. Residual caveat: a +/// future Rust retro-fit re-reading written rows would see the last duplicate +/// value where the live classifier used the first — duplicate rule-keys within +/// one span are pathological, and the caveat is cheaper than the coupling.) +/// +/// The classifier never reads this Map: on the attribute-mapping path it reads +/// [`attr_map_first_wins`] instead, so its verdict does not depend on whether the +/// org happens to have mapping rules configured. fn attr_map(attributes: &[KeyValue]) -> Map { let mut out = Map::with_capacity(attributes.len()); for attribute in attributes { - out.insert( - attribute.key.clone(), - json!(attribute - .value - .as_ref() - .map(any_value_string) - .unwrap_or_default()), - ); + let value = json!(attribute + .value + .as_ref() + .map(any_value_string) + .unwrap_or_default()); + out.insert(attribute.key.clone(), value); + } + out +} + +/// [`attr_map`] under the **classifier's** duplicate rule: the FIRST occurrence of +/// a key wins, matching `ai_classifier`'s dedup of its own view of the wire list. +/// +/// Built only for the span that actually carries a duplicate key on an org that +/// has attribute mappings — the one case where the two rules disagree and the +/// classifier cannot read the wire list directly. Never stored: the row's Map +/// keeps [`attr_map`]'s last-wins canonicalization. +fn attr_map_first_wins(attributes: &[KeyValue]) -> Map { + let mut out = Map::with_capacity(attributes.len()); + for attribute in attributes { + if out.contains_key(&attribute.key) { + continue; + } + let value = json!(attribute + .value + .as_ref() + .map(any_value_string) + .unwrap_or_default()); + out.insert(attribute.key.clone(), value); } out } @@ -2900,6 +3168,13 @@ fn count_log_rows(request: &ExportLogsServiceRequest) -> usize { .sum() } +/// Adversarial classification fixtures. A child of this module, not of `tests`, +/// because it drives [`encode_traces`] — the real row writer — and that is private +/// here. +#[cfg(test)] +#[path = "ai_adversarial_fixtures.rs"] +mod ai_adversarial_fixtures; + #[cfg(test)] mod tests { use super::*; @@ -3310,6 +3585,7 @@ mod tests { &request, &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); assert_eq!(stats.rows, 1); @@ -3626,6 +3902,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ExportDestination::ClickHouse, ) .await @@ -4345,6 +4622,11 @@ mod tests { "links_span_id", "links_trace_state", "links_attributes", + "ai_vendor", + "ai_session_key_state", + "ai_session_key_hash", + "ai_rules_version", + "ai_rollup_hour", ]; const METRIC_COMMON: &[&str] = &[ @@ -4508,6 +4790,374 @@ mod tests { } } + // ----------------------------------------------------------------------- + // AI classification (write-side plan §2/§3) + // ----------------------------------------------------------------------- + + /// Wall-clock instant every AI test clamps against: 2023-11-14 22:13:20 UTC, + /// the same second `populated_trace_request`'s span starts at. + const AI_RECEIVE_SECS: i64 = 1_700_000_000; + + /// spring_ai's only session-key candidate, authoritative on spans with + /// `spring.ai.kind = chat_client`. + const SPRING_AI_SESSION_KEY: &str = "spring.ai.chat.client.conversation.id"; + + /// A Spring AI span: a scope that is only a *conditional* candidate + /// (`org.springframework.boot` is app-chosen, so insufficient on its own), + /// promoted by the `spring.ai.` attribute hit, carrying spring_ai's + /// session-granularity key on an authoritative `chat_client` span. + fn ai_trace_request( + span_attributes: Vec, + start_unix_nano: u64, + ) -> ExportTraceServiceRequest { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![ + string_kv("service.name", "spring-ai-app"), + string_kv("maple_org_id", "org_ai"), + ], + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: "org.springframework.boot".to_string(), + version: "4.1.0".to_string(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + spans: vec![Span { + trace_id: vec![0x44; 16], + span_id: vec![0x55; 8], + name: "chat_client".to_string(), + kind: span::SpanKind::Internal as i32, + start_time_unix_nano: start_unix_nano, + end_time_unix_nano: start_unix_nano + 1_000_000, + attributes: span_attributes, + ..Default::default() + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } + } + + fn spring_ai_attributes() -> Vec { + vec![ + string_kv("spring.ai.kind", "chat_client"), + string_kv("gen_ai.operation.name", "chat"), + string_kv(SPRING_AI_SESSION_KEY, "sess-abc"), + ] + } + + fn encode_ai_row(request: &ExportTraceServiceRequest, ai: &AiClassificationSettings) -> Value { + let (frames, stats) = encode_traces( + &test_cfg().datasources, + "org_ai", + request, + &SamplingPolicy::default(), + &[], + ai, + ) + .unwrap(); + assert_eq!(stats.rows, 1); + frame_row(&frames[0]) + } + + #[test] + fn classification_on_writes_vendor_state_hash_and_version() { + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let row = encode_ai_row(&request, &ai); + + assert_eq!(row["ai_vendor"], "spring_ai"); + assert_eq!( + row["ai_session_key_state"], + json!(ai_classifier::session_state::SESSION) + ); + assert_eq!(row["ai_rules_version"], json!(registry().version())); + // The construction the SQL leg reproduces: cityHash64 of the bare value. + let expected = crate::cityhash102::city_hash64(b"sess-abc"); + assert_ne!(expected, 0); + assert_eq!(row["ai_session_key_hash"], json!(expected)); + assert_eq!(row["ai_rollup_hour"], "2023-11-14 22:00:00"); + } + + #[test] + fn classification_examined_count_matches_row_count() { + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let (_, on) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + &[], + &AiClassificationSettings::at(true, AI_RECEIVE_SECS), + ) + .unwrap(); + assert_eq!(on.ai_spans_examined, on.rows); + + let (_, off) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + &[], + &AiClassificationSettings::at(false, AI_RECEIVE_SECS), + ) + .unwrap(); + assert_eq!(off.ai_spans_examined, 0); + } + + #[test] + fn classification_off_writes_zeros_but_still_writes_the_rollup_hour() { + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let ai = AiClassificationSettings::at(false, AI_RECEIVE_SECS); + let row = encode_ai_row(&request, &ai); + + assert_eq!(row["ai_vendor"], ""); + assert_eq!(row["ai_session_key_state"], json!(0)); + assert_eq!(row["ai_session_key_hash"], json!(0)); + // 0, not the registry version: a flag-off row is "never examined". + assert_eq!(row["ai_rules_version"], json!(0)); + assert_eq!(row["ai_rollup_hour"], "2023-11-14 22:00:00"); + } + + #[test] + fn a_non_ai_span_is_examined_and_stamped_but_carries_no_vendor() { + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let (frames, stats) = encode_traces( + &test_cfg().datasources, + "org_contract", + &populated_trace_request(), + &SamplingPolicy::default(), + &[], + &ai, + ) + .unwrap(); + let row = frame_row(&frames[0]); + assert_eq!(stats.ai_spans_examined, 1); + assert_eq!(row["ai_vendor"], ""); + assert_eq!(row["ai_session_key_state"], json!(0)); + // Non-zero version with an empty vendor is the plan's "definitively + // classified non-AI" marker — the whole point of stamping non-AI spans. + assert_eq!(row["ai_rules_version"], json!(registry().version())); + } + + #[test] + fn rollup_hour_clamps_stale_and_future_timestamps_to_receive_time() { + let receive_hour = "2023-11-14 22:00:00"; + let ai = || AiClassificationSettings::at(true, AI_RECEIVE_SECS); + + // In window, one hour before receive: the span's own hour is kept. + let in_window = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 - 3_600) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&in_window, &ai())["ai_rollup_hour"], + "2023-11-14 21:00:00" + ); + + // In window, at the far edge of the past half (exactly 7 days back). + let edge_past = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 - 7 * 86_400) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&edge_past, &ai())["ai_rollup_hour"], + "2023-11-07 22:00:00" + ); + + // Older than 7 days: clamped to the receive hour. + let too_old = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 - 8 * 86_400) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&too_old, &ai())["ai_rollup_hour"], + receive_hour + ); + + // Further ahead than 1 day: clamped too. This is the replay/attacker case. + let too_new = ai_trace_request( + spring_ai_attributes(), + (AI_RECEIVE_SECS as u64 + 86_400 + 3_600) * 1_000_000_000, + ); + assert_eq!( + encode_ai_row(&too_new, &ai())["ai_rollup_hour"], + receive_hour + ); + + // A zero timestamp is 1970 — far outside the window, so it clamps too + // rather than creating a 1970 partition. + let epoch_zero = ai_trace_request(spring_ai_attributes(), 0); + assert_eq!( + encode_ai_row(&epoch_zero, &ai())["ai_rollup_hour"], + receive_hour + ); + } + + #[test] + fn rollup_hour_matches_the_clickhouse_datetime_format() { + // `DateTime('UTC')` parses `YYYY-MM-DD HH:MM:SS` — 19 chars, no + // fractional part (that is `format_timestamp_nano`'s DateTime64(9)). + let request = ai_trace_request( + spring_ai_attributes(), + AI_RECEIVE_SECS as u64 * 1_000_000_000, + ); + let row = encode_ai_row( + &request, + &AiClassificationSettings::at(false, AI_RECEIVE_SECS), + ); + let hour = row["ai_rollup_hour"].as_str().unwrap(); + assert_eq!(hour.len(), 19, "not DateTime('UTC'): {hour:?}"); + assert!(hour.ends_with(":00:00"), "not an hour boundary: {hour:?}"); + let parsed = chrono::NaiveDateTime::parse_from_str(hour, "%Y-%m-%d %H:%M:%S") + .expect("ClickHouse DateTime literal must round-trip"); + assert_eq!(parsed.and_utc().timestamp() % 3600, 0); + // The generated insert mapping declares the leaf, so ClickHouse's + // `input()` schema is what parses this string. + let traces = clickhouse_insert_mappings::DATASOURCES + .iter() + .find(|mapping| mapping.datasource == "traces") + .unwrap(); + assert!(traces.columns.contains(&"AiRollupHour")); + assert!(traces + .input_schema + .contains("ai_rollup_hour DateTime('UTC')")); + assert!(traces.columns.contains(&"AiVendor")); + assert!(traces.columns.contains(&"AiSessionKeyState")); + assert!(traces.columns.contains(&"AiSessionKeyHash")); + assert!(traces.columns.contains(&"AiRulesVersion")); + } + + #[test] + fn classification_reads_the_row_after_attribute_remapping() { + // The org moves a custom key onto spring_ai's session-key attribute. + // Classifying the wire attributes would miss it and write state 3; + // classifying the remapped shape resolves it. + let attributes = vec![ + string_kv("spring.ai.kind", "chat_client"), + string_kv("app.conversation", "sess-remapped"), + ]; + let request = ai_trace_request(attributes, AI_RECEIVE_SECS as u64 * 1_000_000_000); + let rules = [AttributeMappingRule { + source_context: MappingSourceContext::Span, + source_key: "app.conversation".to_string(), + target_key: SPRING_AI_SESSION_KEY.to_string(), + operation: MappingOperation::Move, + }]; + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let (frames, _) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + &rules, + &ai, + ) + .unwrap(); + let row = frame_row(&frames[0]); + + assert_eq!( + row["span_attributes"][SPRING_AI_SESSION_KEY], + "sess-remapped" + ); + assert_eq!( + row["ai_session_key_state"], + json!(ai_classifier::session_state::SESSION) + ); + assert_eq!( + row["ai_session_key_hash"], + json!(crate::cityhash102::city_hash64(b"sess-remapped" + )) + ); + } + + /// Determinism contract: the classifier's first-occurrence-wins duplicate rule + /// must not depend on whether the org has attribute mappings configured. + /// + /// The row writer's Map is last-wins by design, and with any mapping rule + /// present the classifier reads a list rebuilt from a Map rather than the wire + /// list. Before the first-wins view, two orgs sending byte-identical spans got + /// different `AiVendor`, `AiSessionKeyHash` and rollup rows because one of them + /// happened to have a rule configured — here a `Move` of a key no span carries, + /// which touches nothing. + #[test] + fn duplicate_keys_classify_the_same_with_and_without_mapping_rules() { + let duplicated = vec![ + string_kv("gen_ai.system", "spring_ai"), + string_kv("gen_ai.system", "strands-agents"), + string_kv("gen_ai.operation.name", "chat"), + ]; + let request = ai_trace_request(duplicated, AI_RECEIVE_SECS as u64 * 1_000_000_000); + let ai = AiClassificationSettings::at(true, AI_RECEIVE_SECS); + let no_op_rule = [AttributeMappingRule { + source_context: MappingSourceContext::Span, + source_key: "no.such.key".to_string(), + target_key: "no.such.target".to_string(), + operation: MappingOperation::Move, + }]; + + let classify = |rules: &[AttributeMappingRule]| { + let (frames, _) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + rules, + &ai, + ) + .unwrap(); + let row = frame_row(&frames[0]); + ( + row["ai_vendor"].clone(), + row["ai_session_key_state"].clone(), + row["ai_session_key_hash"].clone(), + ) + }; + + let without = classify(&[]); + let with = classify(&no_op_rule); + // The first duplicate wins on both paths, not the last one. + assert_eq!(without.0, "spring_ai"); + assert_eq!( + with, without, + "an unrelated mapping rule changed the verdict: the remapped view is \ + not being built first-occurrence-wins" + ); + + // The stored Map stays last-wins on both paths — this is a classifier + // contract, not a change to what the row records. + for rules in [&[][..], &no_op_rule[..]] { + let (frames, _) = encode_traces( + &test_cfg().datasources, + "org_ai", + &request, + &SamplingPolicy::default(), + rules, + &ai, + ) + .unwrap(); + assert_eq!( + frame_row(&frames[0])["span_attributes"]["gen_ai.system"], + "strands-agents" + ); + } + } + fn one_of_each_metric_request() -> ExportMetricsServiceRequest { let base = NumberDataPoint { attributes: vec![string_kv("route", "/checkout")], @@ -4639,6 +5289,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); let row = frame_row(&frames[0]); @@ -4714,6 +5365,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); let trace_row = frame_row(&trace_frames[0]); @@ -4768,6 +5420,7 @@ mod tests { &populated_trace_request(), &SamplingPolicy::default(), &[], + &AiClassificationSettings::disabled(), ) .unwrap(); assert_eq!(trace_frames[0].datasource, "tenant_traces_v2"); @@ -4946,7 +5599,13 @@ mod tests { let request = populated_trace_request(); let stats = pipeline - .accept_traces("org_contract", &request, &SamplingPolicy::default(), &[]) + .accept_traces( + "org_contract", + &request, + &SamplingPolicy::default(), + &[], + &AiClassificationSettings::disabled(), + ) .await .unwrap(); assert_eq!(stats.rows, 1); diff --git a/packages/domain/src/ai/hash-alignment.clickhouse.e2e.test.ts b/packages/domain/src/ai/hash-alignment.clickhouse.e2e.test.ts new file mode 100644 index 000000000..82340f14f --- /dev/null +++ b/packages/domain/src/ai/hash-alignment.clickhouse.e2e.test.ts @@ -0,0 +1,176 @@ +// The hash contract: `AiSessionKeyHash` is the same 64 bits in Rust and in ClickHouse. +// +// This is not an equivalence suite — no SQL re-derives a classification anywhere. It +// proves one narrow, load-bearing fact: `cityHash64(x)` evaluated by ClickHouse equals +// `city_hash64(x)` computed by the ingest writer, for every byte string. The read path's +// computability claim rests on exactly that — the plaintext session key stays in +// `SpanAttributes` on the same row, so `WHERE AiSessionKeyHash = cityHash64({sessionId})` +// finds a known session through the indexed column, and the column is verifiable and +// repairable from `SpanAttributes` in SQL. If the two hashes ever diverge, both stop +// being true and nothing else in the stack notices. +// +// The variant is the load-bearing part: ClickHouse vendors CityHash **1.0.2**, and +// CityHash 1.1 changed the mixing for inputs ≤ 32 bytes and > 64 bytes. That is why +// `apps/ingest/src/cityhash102.rs` is a frozen in-crate port rather than a crate +// dependency, and why this suite exercises those length bands. +// +// Vectors come from the adversarial fixture's `session_key_hex` — the raw winning +// session-key value, which no row carries — so the shapes under test are the hostile +// ones the classifier actually resolved: astral-plane UTF-8, embedded NULs, quotes and +// backslashes, and a 64 KiB value. Values are passed as `unhex(...)` rather than string +// literals so those bytes reach the server exactly as Rust hashed them; a +// literal-escaping bug would otherwise surface as a hash mismatch and be misread as a +// variant mismatch. One ASCII vector is additionally checked as a plain literal to prove +// `unhex` is not itself the thing under test. +// +// bun ch:up +// CLICKHOUSE_E2E=1 bun run --cwd packages/domain test -- hash-alignment.clickhouse.e2e + +import { existsSync, readFileSync } from "node:fs" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" +import { describe, expect, it } from "vitest" + +const clickhouseE2eEnabled = process.env.CLICKHOUSE_E2E === "1" +const clickhouseUrl = process.env.CLICKHOUSE_E2E_URL ?? "http://127.0.0.1:8123" +const clickhouseUser = process.env.CLICKHOUSE_E2E_USER ?? "maple" +const clickhousePassword = process.env.CLICKHOUSE_E2E_PASSWORD ?? "maple" + +/** + * Managed Tinybird is ClickHouse 24.12 with `use_variant_as_common_type = 0`, where a + * type mismatch between branches is a hard error; modern servers default it on and + * quietly resolve the same expression to a `Variant`. Pinned for the same reason the + * apps/api harness pins it. + */ +const ANALYZER_STRICTNESS: Readonly> = { use_variant_as_common_type: "0" } + +const clickhouseExec = async (sql: string): Promise => { + const query = new URLSearchParams({ database: "default", ...ANALYZER_STRICTNESS }) + const response = await fetch(`${clickhouseUrl.replace(/\/$/, "")}/?${query.toString()}`, { + method: "POST", + redirect: "manual", + headers: { + "Content-Type": "text/plain", + "X-ClickHouse-User": clickhouseUser, + "X-ClickHouse-Key": clickhousePassword, + "X-ClickHouse-Database": "default", + }, + body: sql, + }) + const body = await response.text() + if (!response.ok) throw new Error(`ClickHouse ${response.status}: ${body.slice(0, 1500)}`) + return body +} + +const clickhouseSelect = async (sql: string): Promise> => { + const body = await clickhouseExec(`${sql}\nFORMAT JSONEachRow`) + return body + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => JSON.parse(line) as A) +} + +/** Walk up to the workspace root rather than counting `../` segments. */ +const repoRoot = ((): string => { + let dir = dirname(fileURLToPath(import.meta.url)) + while (!existsSync(join(dir, "turbo.json"))) { + const parent = dirname(dir) + if (parent === dir) + throw new Error("Could not locate the workspace root: no turbo.json above this file") + dir = parent + } + return dir +})() + +interface FixtureRecord { + readonly id: string + readonly rust?: { + /** A decimal string: a UInt64 hash above 2^53 does not survive `JSON.parse`. */ + readonly session_key_hash: string + /** Hex of the raw winning session-key value, or null below state 5. */ + readonly session_key_hex: string | null + } +} + +interface HashVector { + readonly id: string + readonly valueHex: string + readonly expected: string +} + +const vectors: ReadonlyArray = readFileSync( + join(repoRoot, "apps/ingest/fixtures/adversarial/adversarial-spans.jsonl"), + "utf8", +) + .split("\n") + .filter((line) => line.trim().length > 0) + .flatMap((line) => { + const record = JSON.parse(line) as FixtureRecord + const valueHex = record.rust?.session_key_hex + if (valueHex === null || valueHex === undefined) return [] + return [{ id: record.id, valueHex, expected: record.rust?.session_key_hash ?? "" }] + }) + +const decode = (hex: string): string => Buffer.from(hex, "hex").toString("utf8") + +describe.skipIf(!clickhouseE2eEnabled)("AiSessionKeyHash Rust↔ClickHouse hash contract", () => { + it("covers the adversarial value shapes, not just ASCII", () => { + // A guard on the guard: if the fixture ever stops producing resolved session keys + // with these shapes, the rest of this suite silently proves much less. + expect(vectors.length).toBeGreaterThan(40) + const values = vectors.map((vector) => decode(vector.valueHex)) + expect(values.some((value) => value.includes(" "))).toBe(true) + expect(values.some((value) => /[\u{10000}-\u{10FFFF}]/u.test(value))).toBe(true) + expect(values.some((value) => value.length > 60_000)).toBe(true) + expect(values.some((value) => value.includes("'") || value.includes("\\"))).toBe(true) + expect(values.some((value) => value.includes("\0"))).toBe(true) + }) + + it("computes cityHash64(value) identically to the Rust writer", async () => { + const rows = vectors + .map((vector) => `('${vector.id.replace(/'/g, "''")}', '${vector.valueHex}')`) + .join(",\n\t") + const results = await clickhouseSelect<{ readonly id: string; readonly hash: string }>( + `SELECT + id, + toString(cityHash64(unhex(value_hex))) AS hash +FROM values('id String, value_hex String', + ${rows} +)`, + ) + + expect(results).toHaveLength(vectors.length) + const byId = new Map(results.map((row) => [row.id, row.hash])) + const mismatches = vectors + .filter((vector) => byId.get(vector.id) !== vector.expected) + .map( + (vector) => + `${vector.id}: rust=${vector.expected} clickhouse=${byId.get(vector.id)} value=${JSON.stringify(decode(vector.valueHex).slice(0, 120))}`, + ) + expect(mismatches, `hash construction diverges:\n${mismatches.join("\n")}`).toEqual([]) + }) + + it("agrees when the same bytes are written as a plain SQL literal", async () => { + // `unhex` is the transport, not the claim. One pure-ASCII vector, spelled out. + const vector = vectors.find((candidate) => decode(candidate.valueHex) === "sess-claude") + expect(vector, "fixture no longer contains the ASCII control vector").toBeDefined() + const body = await clickhouseExec(`SELECT toString(cityHash64('sess-claude')) FORMAT TabSeparated`) + expect(body.trim()).toBe((vector as HashVector).expected) + }) + + it("pins the negative: multi-argument cityHash64 is a different function", async () => { + // Why any SQL that recomputes this column must pass exactly one argument. + // `cityHash64(a, b)` combines per-argument hashes rather than hashing the + // concatenation, so a query written that way would never match a stored column. + // The landmine `cityhash102.rs`'s module doc warns about, pinned on the SQL side. + const body = await clickhouseExec( + `SELECT + toString(cityHash64('ab')) AS single, + toString(cityHash64('a', 'b')) AS multi +FORMAT TabSeparated`, + ) + const [single, multi] = body.trim().split("\t") + expect(single).toBe("1725057946192985918") + expect(multi).not.toBe(single) + }) +}) From ee03d7324f3af1233d5f1e1efb1f17b2a583f47a Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 13 Aug 2026 12:28:23 +0200 Subject: [PATCH 3/3] feat(cli): carry the AI classification columns into the local store Local schema v5: the same five defaulted columns and two skip indexes on `traces`, no new objects. The v4 -> v5 module and a frozen v5 DDL snapshot keep an existing local store readable after the generated current schema advances, and the manifest gate now checks that snapshot's identity the way it already checks v1 through v4. The local OTLP encoder stamps the same five fields, so a local store and the hosted warehouse hold the same shape for the same span. Asserted as a column and index delta against the frozen v4 manifest rather than a whole-manifest snapshot, so a stray table or a rewritten column cannot ride along on this version. Co-Authored-By: Claude Fable 5 --- apps/cli/src/server/local-schema-history.ts | 7 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 7 + .../v5-to-v6-ai-classification-columns.ts | 247 +++ apps/cli/src/server/otlp/encode.test.ts | 51 + apps/cli/src/server/otlp/encode.ts | 54 + apps/cli/src/server/schema-identity.ts | 17 +- apps/cli/src/server/schema/local-inserts.json | 18 +- .../cli/src/server/schema/local-schema-v6.sql | 1746 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 13 +- apps/cli/test/local-store-migrations.test.ts | 58 +- scripts/check-local-schema-manifest.ts | 15 + 12 files changed, 2217 insertions(+), 18 deletions(-) create mode 100644 apps/cli/src/server/local-store-migrations/v5-to-v6-ai-classification-columns.ts create mode 100644 apps/cli/src/server/schema/local-schema-v6.sql diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 775481ca8..a39f0f5ac 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -58,4 +58,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "9d3b16f4f882049d40cf5bb31b9224243fcdade009c34b833212788f8cd9cc1d", projectRevision: "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7", }), + Object.freeze({ + version: 6, + fingerprint: "daa45b39f38c7655", + digest: "daa45b39f38c7655c074781cd77dce68e90b60a175461197dcdb8bc8a13088a1", + manifestDigest: "1849d5063a8e88b75dbd40bc4bae46f380a562ffcee5705865b0568b5ff04b40", + projectRevision: "3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 40530abf3..9ea99d34f 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 5 as const +export const LOCAL_SCHEMA_VERSION = 6 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 0e9e6fbe8..ae19364d3 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -37,6 +37,7 @@ import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" import { v4ToV5ServiceOverviewMinutelyModule } from "./local-store-migrations/v4-to-v5-service-overview-minutely" +import { v5ToV6AiClassificationColumnsModule } from "./local-store-migrations/v5-to-v6-ai-classification-columns" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -56,6 +57,11 @@ export { } from "./local-store-migration-module" export { legacyToCurrentModule } from "./local-store-migrations/legacy-to-current" +export { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error-rollup" +export { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" +export { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-events" +export { v4ToV5ServiceOverviewMinutelyModule } from "./local-store-migrations/v4-to-v5-service-overview-minutely" +export { v5ToV6AiClassificationColumnsModule } from "./local-store-migrations/v5-to-v6-ai-classification-columns" const NONTERMINAL_PHASES = new Set([ "planned", @@ -121,6 +127,7 @@ export const localStoreMigrations: ReadonlyArray = v2ToV3ServiceMapIngestBridgeModule, v3ToV4WebEventsModule, v4ToV5ServiceOverviewMinutelyModule, + v5ToV6AiClassificationColumnsModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v5-to-v6-ai-classification-columns.ts b/apps/cli/src/server/local-store-migrations/v5-to-v6-ai-classification-columns.ts new file mode 100644 index 000000000..9cadad047 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v5-to-v6-ai-classification-columns.ts @@ -0,0 +1,247 @@ +import { AI_CLASSIFICATION_ALTER_STATEMENTS } from "@maple/domain/clickhouse" +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V5_SQL, + LOCAL_SCHEMA_V6, + LOCAL_SCHEMA_V6_MANIFEST, + LOCAL_SCHEMA_V6_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +interface V5ToV6State { + readonly module: "local-0005-to-0006-ai-classification-columns" + readonly version: 1 + readonly rawRows: Readonly> + readonly retentionDays?: number +} + +interface V5ToV6Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v5 -> v6 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (typeof count !== "string" || !/^\d+$/.test(count)) + throw new Error(`v5 -> v6 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v5 -> v6 rawRows contains an unknown table") + return counts +} + +const decodeState = (value: unknown): V5ToV6State => { + if (!isRecord(value)) throw new Error("v5 -> v6 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v5 -> v6 state contains an unknown field") + if (value.module !== "local-0005-to-0006-ai-classification-columns" || value.version !== 1) + throw new Error("v5 -> v6 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v5 -> v6 retentionDays must be an integer") + return { + module: "local-0005-to-0006-ai-classification-columns", + version: 1, + rawRows: decodeCounts(value.rawRows), + ...(value.retentionDays === undefined ? {} : { retentionDays: value.retentionDays }), + } +} + +const decodeProgress = (value: unknown): V5ToV6Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v5 -> v6 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V5_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V5_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, + ) + return { + module: "local-0005-to-0006-ai-classification-columns", + version: 1, + rawRows, + ...(retentionDays === undefined ? {} : { retentionDays }), + } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V5ToV6State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Runs ClickHouse migration 0016's ALTER list — the imported constant, not a + * copy of it, so retuning an index or adding a column there reaches migrated + * local stores too. `traces` already exists here, so bootstrapping the v6 DDL is + * a no-op on it (`CREATE TABLE IF NOT EXISTS`) and the columns and indexes + * arrive only through these ALTERs. + * + * Metadata-only and additive: five trailing columns with constant DEFAULTs plus + * two skip indexes on `traces`. No part is rewritten — existing parts read the + * defaults for free, and every ALTER is `IF NOT EXISTS`, so a resumed run + * converges instead of failing. + * + * Deliberately no `MATERIALIZE INDEX`: that is a mutation over the whole table, + * and the store's 30-day raw-telemetry TTL rolls every unindexed part out on its + * own. Same reasoning as ClickHouse migration 0016 — the local store is not a + * different decision, just a smaller one. + * + * Nothing writes the new columns yet (the ingest classifier lands in a later + * stage), so the migrated store's `traces` rows all carry the defaults: AiVendor + * '' = not classified, AiRulesVersion 0 = the row predates classification. + */ +const apply = async (context: MigrationModuleContext): Promise => + context.openTarget( + (db) => { + for (const statement of AI_CLASSIFICATION_ALTER_STATEMENTS) db.exec(statement) + return { installed: true } as const + }, + { schemaSql: LOCAL_SCHEMA_V6_SQL, bootstrapSchema: true }, + ) + +const verify = async ( + context: MigrationModuleContext, + state: V5ToV6State, + _progress: V5ToV6Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V6_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v5 -> v6 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V6_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v5-store", + description: "Clone the stopped v5 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "add-ai-classification-columns", + description: "Add the AI classification columns and vendor/scope skip indexes to traces", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v6-schema", + description: "Verify the v6 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v5 store is cloned byte-for-byte before additive DDL runs.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "Trailing defaulted columns and skip indexes are metadata-only; no existing row or part is rewritten and every prior column keeps its value.", + }, + { + // The new columns are readable immediately — every pre-migration row reads + // its DEFAULT — but they are not *classified*: nothing writes them until the + // ingest classifier ships, and AiRulesVersion = 0 is precisely the marker + // for "this row was never examined". The skip indexes cover only parts + // written after the ALTER; the 30-day TTL retires the rest. + name: "traces AI classification columns", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Defaulted for every existing row (AiRulesVersion 0 = never classified) and filled forward once the ingest classifier ships; complete by construction after the 30-day raw-telemetry horizon.", + preservationInterval: "traces retention horizon", + sourceRetentionDays: 30, + targetRetentionDays: 30, + }, +] + +export const v5ToV6AiClassificationColumnsModule: LocalStoreMigrationModule = { + id: "local-0005-to-0006-ai-classification-columns", + moduleVersion: 1, + description: "Add the AI classification columns and vendor/scope skip indexes to traces on v5", + from: LOCAL_SCHEMA_V5, + to: LOCAL_SCHEMA_V6, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/otlp/encode.test.ts b/apps/cli/src/server/otlp/encode.test.ts index 10ce95c01..433f94fc2 100644 --- a/apps/cli/src/server/otlp/encode.test.ts +++ b/apps/cli/src/server/otlp/encode.test.ts @@ -6,6 +6,7 @@ import { encodeLogs, encodeMetrics, encodeTraces, + formatRollupHour, formatTimestampNano, OtlpFieldError, spanIdHex, @@ -257,6 +258,56 @@ describe("value-level spot checks", () => { }) }) +// `formatRollupHour` is a hand-port of Rust `rollup_hour_secs` + +// `format_datetime_secs` (`apps/ingest/src/telemetry.rs`), and a hand-port with no +// boundary tests is a guess. The receive time and the expectations below are the +// same ones `rollup_hour_clamps_stale_and_future_timestamps_to_receive_time` +// asserts on the Rust side, so a divergence shows up as one of these failing +// rather than as local stores partitioning differently from the gateway. +describe("formatRollupHour clamp (port of the Rust rollup_hour_secs)", () => { + const RECEIVE_SECS = 1_700_000_000 // 2023-11-14 22:13:20 UTC + const RECEIVE_HOUR = "2023-11-14 22:00:00" + const DAY = 86_400 + const nanos = (secs: number) => (BigInt(secs) * 1_000_000_000n).toString() + const at = (secs: number) => formatRollupHour(nanos(secs), RECEIVE_SECS) + + it("keeps the span's own hour inside the window, including both edges", () => { + // The window is inclusive at both ends: `>=` past, `<=` future. + expect(at(RECEIVE_SECS - 3_600)).toBe("2023-11-14 21:00:00") + expect(at(RECEIVE_SECS - 7 * DAY)).toBe("2023-11-07 22:00:00") // exactly −7d + expect(at(RECEIVE_SECS + DAY)).toBe("2023-11-15 22:00:00") // exactly +1d + }) + + it("clamps to the receive hour one second outside either edge", () => { + // One second, not one hour: an off-by-one in the comparison would survive a + // coarser probe, and the past edge is exactly where a legitimate late + // backfill turns into unbounded partition creation. + expect(at(RECEIVE_SECS - 7 * DAY - 1)).toBe(RECEIVE_HOUR) + expect(at(RECEIVE_SECS + DAY + 1)).toBe(RECEIVE_HOUR) + // The replay/attacker case the clamp exists for, and the zero timestamp a + // span with no start time produces — 1970 is outside the window, so it + // clamps rather than creating a 1970 partition. + expect(at(RECEIVE_SECS + 30 * DAY)).toBe(RECEIVE_HOUR) + expect(formatRollupHour("0", RECEIVE_SECS)).toBe(RECEIVE_HOUR) + // The TS side alone has to survive a missing or unparseable field, where + // Rust has a `u64`. Both fall back to 0, hence to the receive hour. + expect(formatRollupHour(undefined, RECEIVE_SECS)).toBe(RECEIVE_HOUR) + expect(formatRollupHour("not-a-number", RECEIVE_SECS)).toBe(RECEIVE_HOUR) + }) + + it("renders DateTime('UTC'), not DateTime64(9)", () => { + // 19 chars on an hour boundary, no fractional part — that is + // `format_datetime_secs`, and it is what ClickHouse's + // `input('… AiRollupHour DateTime(\'UTC\')')` parses. + const hour = at(RECEIVE_SECS) + expect(hour).toHaveLength(19) + expect(hour).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:00:00$/) + expect((Date.parse(`${hour.replace(" ", "T")}Z`) / 1000) % 3600).toBe(0) + // The epoch itself, where the Rust `None` branch returns the same literal. + expect(formatRollupHour("0", 0)).toBe("1970-01-01 00:00:00") + }) +}) + describe("OTLP/JSON hex ids", () => { const spanHex = "b7ad6b7169203331" diff --git a/apps/cli/src/server/otlp/encode.ts b/apps/cli/src/server/otlp/encode.ts index 32cd105a0..68647264d 100644 --- a/apps/cli/src/server/otlp/encode.ts +++ b/apps/cli/src/server/otlp/encode.ts @@ -202,6 +202,46 @@ export function formatTimestampNano(nanos: string | number | undefined): string return `${calendar}.${frac9}` } +/** Seconds in the clamp window's past and future halves (write-side plan §3). */ +const ROLLUP_CLAMP_PAST_SECS = 7 * 24 * 60 * 60 +const ROLLUP_CLAMP_FUTURE_SECS = 24 * 60 * 60 + +/** + * Port of Rust `rollup_hour_secs` + `format_datetime_secs` (see + * `apps/ingest/src/telemetry.rs`). `AiRollupHour` is `toStartOfHour(start time)` + * when the span's start is within `[receive − 7d, receive + 1d]`, else + * `toStartOfHour(receive time)`, rendered as `DateTime('UTC')`'s + * `YYYY-MM-DD HH:MM:SS`. + * + * Local mode has no classifier, so the other four AI columns stay at their + * "never examined" defaults — but this one is written for real on every span, + * exactly as the gateway does. It is `service_ai_vendors_hourly`'s partition + * key, so an unclamped client timestamp means unbounded partition creation and + * rows whose TTL never fires. Clamping at write time is also what keeps a later + * partition rebuild from relocating rows: an MV-side clamp would need `now()`, + * re-evaluated at rebuild time. + */ +export function formatRollupHour( + spanStartUnixNano: string | number | undefined, + receiveTimeSecs: number, +): string { + let spanSecs = 0 + try { + spanSecs = Number(BigInt(spanStartUnixNano ?? 0) / 1_000_000_000n) + } catch { + spanSecs = 0 + } + const inWindow = + Number.isFinite(spanSecs) && + spanSecs >= receiveTimeSecs - ROLLUP_CLAMP_PAST_SECS && + spanSecs <= receiveTimeSecs + ROLLUP_CLAMP_FUTURE_SECS + const chosen = inWindow ? spanSecs : receiveTimeSecs + const hour = chosen - (((chosen % 3600) + 3600) % 3600) + const date = new Date(hour * 1000) + if (Number.isNaN(date.getTime())) return "1970-01-01 00:00:00" + return date.toISOString().slice(0, 19).replace("T", " ") +} + /** * Port of Rust `any_value_string`: coerce an OTLP `AnyValue` to a string * exactly as the Rust encoder does. @@ -542,6 +582,9 @@ interface Span { export function encodeTraces(req: unknown): EncodedBatch[] { const request = (req ?? {}) as TraceRequest const rows: Record[] = [] + // One receive time for the whole batch, matching the gateway: two spans in + // the same request must not be able to land on different clamp anchors. + const receiveTimeSecs = Math.floor(Date.now() / 1000) for (const resourceSpans of request.resourceSpans ?? []) { const resourceAttrs = attrMap(resourceSpans.resource?.attributes) @@ -588,6 +631,17 @@ export function encodeTraces(req: unknown): EncodedBatch[] { links_span_id: links.map((link, i) => spanIdHex(link.spanId, `span.links[${i}].spanId`)), links_trace_state: links.map((link) => link.traceState ?? ""), links_attributes: links.map((link) => attrMap(link.attributes)), + // Local mode runs no classifier, so these four stay at the + // "never examined" defaults the ClickHouse columns declare: + // AiVendor '' = not classified as AI, AiRulesVersion 0 = never + // looked at. `service_ai_vendors_hourly` filters on AiVendor != '', + // so a local store's rollup stays empty rather than reporting a + // confident zero — which is the honest answer here. + ai_vendor: "", + ai_session_key_state: 0, + ai_session_key_hash: 0, + ai_rules_version: 0, + ai_rollup_hour: formatRollupHour(span.startTimeUnixNano, receiveTimeSecs), }) } } diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index aedf4d40a..629bf0519 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -4,6 +4,7 @@ import schemaV2Sql from "./schema/local-schema-v2.sql" with { type: "text" } import schemaV3Sql from "./schema/local-schema-v3.sql" with { type: "text" } import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } +import schemaV6Sql from "./schema/local-schema-v6.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -27,7 +28,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7" + "3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -63,6 +64,11 @@ export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest +/** Immutable v6 DDL/manifest snapshot used by the v5 -> v6 module after the + * generated current schema advances. */ +export const LOCAL_SCHEMA_V6_SQL = schemaV6Sql +export const LOCAL_SCHEMA_V6_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV6Sql) +export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = LOCAL_SCHEMA_V6_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -122,6 +128,15 @@ export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, }) +export const LOCAL_SCHEMA_V6: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_HISTORY[6]!.version, + fingerprint: LOCAL_SCHEMA_HISTORY[6]!.fingerprint, + digest: LOCAL_SCHEMA_HISTORY[6]!.digest, + manifestDigest: LOCAL_SCHEMA_HISTORY[6]!.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: LOCAL_SCHEMA_HISTORY[6]!.projectRevision, +}) + export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, fingerprint: SCHEMA_FINGERPRINT, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index dbb699904..64d8287c1 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7", + "projectRevision": "3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349", "orgPlaceholder": "__ORG__", "datasources": { "traces": { @@ -30,7 +30,12 @@ "LinksTraceId", "LinksSpanId", "LinksTraceState", - "LinksAttributes" + "LinksAttributes", + "AiVendor", + "AiSessionKeyState", + "AiSessionKeyHash", + "AiRulesVersion", + "AiRollupHour" ], "selects": [ "__ORG__", @@ -58,9 +63,14 @@ "links_trace_id", "links_span_id", "links_trace_state", - "links_attributes" + "links_attributes", + "ai_vendor", + "ai_session_key_state", + "ai_session_key_hash", + "ai_rules_version", + "ai_rollup_hour" ], - "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))" + "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String)), ai_vendor LowCardinality(String), ai_session_key_state UInt8, ai_session_key_hash UInt64, ai_rules_version UInt32, ai_rollup_hour DateTime('UTC')" }, "logs": { "table": "logs", diff --git a/apps/cli/src/server/schema/local-schema-v6.sql b/apps/cli/src/server/schema/local-schema-v6.sql new file mode 100644 index 000000000..5d212f3f3 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v6.sql @@ -0,0 +1,1746 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349 +-- localSchemaVersion: 6 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + AiVendor LowCardinality(String) DEFAULT '', + AiSessionKeyState UInt8 DEFAULT 0, + AiSessionKeyHash UInt64 DEFAULT 0, + AiRulesVersion UInt32 DEFAULT 0, + AiRollupHour DateTime('UTC') DEFAULT toDateTime(0), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4, + INDEX idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + arraySlice( + arrayFilter( + line -> match(line, ':[0-9]+|line [0-9]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + arrayMap( + line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection (only consulted when _fpFrames = '') + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- Fold into the existing fallback hash slot. Non-JSON path is unchanged. + multiIf( + _fpFrames != '', '', + _isJsonObj, _jsonSig, + replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#') + ) AS _msgFallback, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index d47684e8e..5d212f3f3 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 3bf63ab19fcba1a20ebaf6a97b49acfc2ecf00f1589c11438349bb87d21f77b7 --- localSchemaVersion: 5 +-- projectRevision: 3e7d570ffbf917f749dfee58f5920c8cdfbff887366323da7249fc99cce0d349 +-- localSchemaVersion: 6 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -799,13 +799,20 @@ CREATE TABLE IF NOT EXISTS traces ( ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + AiVendor LowCardinality(String) DEFAULT '', + AiSessionKeyState UInt8 DEFAULT 0, + AiSessionKeyHash UInt64 DEFAULT 0, + AiRulesVersion UInt32 DEFAULT 0, + AiRollupHour DateTime('UTC') DEFAULT toDateTime(0), INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_ai_vendor AiVendor TYPE set(0) GRANULARITY 4, + INDEX idx_scope_name ScopeName TYPE tokenbf_v1(4096, 3, 0) GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index a66ab3630..f655d700d 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -14,6 +14,9 @@ import { LOCAL_SCHEMA_V4, LOCAL_SCHEMA_V4_MANIFEST, LOCAL_SCHEMA_V5, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V6, + LOCAL_SCHEMA_V6_MANIFEST, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -55,16 +58,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v5 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("c36c52a95568eb68") - expect(SCHEMA_DIGEST).toBe("c36c52a95568eb68f8ebc98d7d36b552f21fb09b888bb310c68f0ad52d529fe4") + it("matches the generated v6 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("daa45b39f38c7655") + expect(SCHEMA_DIGEST).toBe("daa45b39f38c7655c074781cd77dce68e90b60a175461197dcdb8bc8a13088a1") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(5) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V5) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(6) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V6) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -119,8 +122,43 @@ describe("current local schema identity", () => { expect(minutelyView?.definition).toContain("FROM traces") expect(minutelyView?.definition).not.toContain("FROM service_overview_minutely") expect( - LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name).filter((name) => !v4Names.has(name)), + LOCAL_SCHEMA_V5_MANIFEST.objects + .map((object) => object.name) + .filter((name) => !v4Names.has(name)), ).toEqual(["service_overview_minutely", "service_overview_minutely_mv"]) + + // v6 adds no objects at all — it is five trailing defaulted columns and two + // skip indexes on traces. Asserted as a column/index delta against the + // frozen v5 manifest so a stray table or a rewritten column can't ride along. + const v5Names = new Set(LOCAL_SCHEMA_V5_MANIFEST.objects.map((object) => object.name)) + expect( + LOCAL_SCHEMA_V6_MANIFEST.objects + .map((object) => object.name) + .filter((name) => !v5Names.has(name)), + ).toEqual([]) + const v5Traces = LOCAL_SCHEMA_V5_MANIFEST.objects.find((object) => object.name === "traces") + const traces = LOCAL_SCHEMA_V6_MANIFEST.objects.find((object) => object.name === "traces") + const v5TraceColumns = new Set(v5Traces?.columns.map((column) => column.name)) + expect( + traces?.columns.map((column) => column.name).filter((name) => !v5TraceColumns.has(name)), + ).toEqual(["AiVendor", "AiSessionKeyState", "AiSessionKeyHash", "AiRulesVersion", "AiRollupHour"]) + // Every new column carries a DEFAULT, so a store migrated from v4 reads the + // same values as a fresh v5 for rows written before the classifier existed. + // (A DEFAULT alone does NOT keep a column out of the generated ingest INSERT + // list — that needs a DEFAULT *plus* an identity JSONPath (`$.`), + // the shape of a column ClickHouse computes for itself. All five of these carry + // a snake_case path instead, so they stay in the insert mappings, which is + // correct: the writer stamps them.) + expect( + traces?.columns + .filter((column) => column.name.startsWith("Ai")) + .every((column) => column.defaultKind === "DEFAULT"), + ).toBe(true) + expect(traces?.columns.find((column) => column.name === "AiRollupHour")?.type).toBe("DateTime('UTC')") + expect(traces?.indexes.filter((index) => !(v5Traces?.indexes ?? []).includes(index))).toEqual([ + "idx_ai_vendor", + "idx_scope_name", + ]) }) }) @@ -133,6 +171,7 @@ describe("local migration registry", () => { "local-0002-to-0003-service-map-ingest-bridge", "local-0003-to-0004-web-events", "local-0004-to-0005-service-overview-minutely", + "local-0005-to-0006-ai-classification-columns", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -140,6 +179,7 @@ describe("local migration registry", () => { expect(chain[2]?.to).toEqual(LOCAL_SCHEMA_V3) expect(chain[3]?.to).toEqual(LOCAL_SCHEMA_V4) expect(chain[4]?.to).toEqual(LOCAL_SCHEMA_V5) + expect(chain[5]?.to).toEqual(LOCAL_SCHEMA_V6) expect(typeof chain[0]?.apply).toBe("function") }) @@ -161,12 +201,12 @@ describe("local migration registry", () => { maple: "dev", createdAt: "2026-01-01T00:00:00.000Z", createdByMaple: "dev", - schemaVersion: 4, + schemaVersion: 6, schemaDigest: SCHEMA_DIGEST, schema: SCHEMA_FINGERPRINT, activation: "active", }), - ).toMatchObject({ version: 4, fingerprint: SCHEMA_FINGERPRINT, digest: SCHEMA_DIGEST }) + ).toMatchObject({ version: 6, fingerprint: SCHEMA_FINGERPRINT, digest: SCHEMA_DIGEST }) }) it("rejects unknown, future, downgrade, and ambiguous paths", () => { @@ -178,7 +218,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 6, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 7, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index 328c60c6a..644acbc02 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -19,6 +19,9 @@ import { LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST_DIGEST, LOCAL_SCHEMA_V5_SQL, + LOCAL_SCHEMA_V6, + LOCAL_SCHEMA_V6_MANIFEST_DIGEST, + LOCAL_SCHEMA_V6_SQL, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -119,6 +122,18 @@ if ( fail("the immutable local schema v5 snapshot no longer matches its historical identity") } +const v6 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V6.version) +if ( + !v6 || + LOCAL_SCHEMA_V6_MANIFEST_DIGEST !== v6.manifestDigest || + schemaFingerprint(LOCAL_SCHEMA_V6_SQL) !== v6.fingerprint || + schemaDigest(LOCAL_SCHEMA_V6_SQL) !== v6.digest || + LOCAL_SCHEMA_V6.fingerprint !== v6.fingerprint || + LOCAL_SCHEMA_V6.digest !== v6.digest +) { + fail("the immutable local schema v6 snapshot no longer matches its historical identity") +} + const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name) if (new Set(names).size !== names.length) fail("local structural schema manifest contains duplicate object names")