From 8d25f4210d798d30ac48bc0b512c28c50193ab34 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 7 Aug 2026 15:18:10 -0400 Subject: [PATCH 01/12] refactor(alerting): extract host-neutral core --- apps/api/package.json | 1 + apps/api/src/services/alerts/AlertsService.ts | 394 ++++++------------ bun.lock | 12 + packages/alerting-core/README.md | 28 ++ packages/alerting-core/package.json | 18 + packages/alerting-core/src/index.test.ts | 193 +++++++++ packages/alerting-core/src/index.ts | 321 ++++++++++++++ packages/alerting-core/tsconfig.json | 17 + 8 files changed, 717 insertions(+), 267 deletions(-) create mode 100644 packages/alerting-core/README.md create mode 100644 packages/alerting-core/package.json create mode 100644 packages/alerting-core/src/index.test.ts create mode 100644 packages/alerting-core/src/index.ts create mode 100644 packages/alerting-core/tsconfig.json diff --git a/apps/api/package.json b/apps/api/package.json index 4080a26ee..206780653 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -34,6 +34,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/clickhouse-builder": "workspace:*", "@maple-dev/effect-sdk": "workspace:*", + "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 9eceed529..1b382929b 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -1,3 +1,13 @@ +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + type AlertEvaluation as EvaluatedRule, + type AlertLifecycleInput, +} from "@maple/alerting-core" import { formatWarehouseDateTime } from "@maple/query-engine" import { AlertComparator as AlertComparatorSchema, @@ -127,31 +137,12 @@ const WAREHOUSE_FAILURE_CATEGORIES = { "@maple/http/errors/WarehouseValidationError": "tinybird_validation", } satisfies Record -interface EvaluatedRule { - readonly status: Schema.Schema.Type - readonly value: number | null - readonly sampleCount: number - readonly threshold: number - readonly thresholdUpper: number | null - readonly comparator: AlertComparator - readonly reason: string - /** - * The window returned nothing and `noDataBehavior: "zero"` synthesized the - * value. Such a status is a statement about the absence of data, not about - * the health of the system — a `gt` rule reads a total ingest outage as - * `healthy` this way. Anything that acts on "healthy" destructively (i.e. - * resolving an open incident) must prove telemetry is still flowing first. - */ - readonly derivedFromNoData: boolean -} - interface DeliveryAttemptFailure { readonly message: string readonly kind: "transport" | "timeout" | "payload" | "destination" | "unknown" readonly retryable: boolean } -const MAX_DELIVERY_ATTEMPTS = 5 const ALERT_TEST_DELIVERY_CONCURRENCY = 5 const ALERT_CHECK_INGEST_CONCURRENCY = 4 // Storm fuse: cap issue-hub upserts per scheduler tick so a pathological @@ -238,25 +229,7 @@ const MAX_PREVIEW_BUCKETS = 200 /** Preserve each org's oldest-first order while preventing one org from monopolizing a tick. */ export const interleaveAlertRulesByOrg = ( rows: ReadonlyArray, -): ReadonlyArray => { - const queues = new Map() - for (const row of rows) { - const queue = queues.get(row.orgId) - if (queue) queue.push(row) - else queues.set(row.orgId, [row]) - } - - const fair: T[] = [] - let index = 0 - while (fair.length < rows.length) { - for (const queue of queues.values()) { - const row = queue[index] - if (row !== undefined) fair.push(row) - } - index += 1 - } - return fair -} +): ReadonlyArray => interleaveAlertRulesByTenant(rows, (row) => row.orgId) // Tinybird DateTime64(3) wire format for alert_checks ingest: // "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). @@ -266,27 +239,6 @@ const toIngestDateTime64 = (epochMs: number) => { return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` } -const compareThreshold = ( - value: number, - comparator: AlertComparator, - threshold: number, - thresholdUpper: number | null = null, -): boolean => - Match.value(comparator).pipe( - Match.when("gt", () => value > threshold), - Match.when("gte", () => value >= threshold), - Match.when("lt", () => value < threshold), - Match.when("lte", () => value <= threshold), - Match.when("eq", () => value === threshold), - Match.when("neq", () => value !== threshold), - Match.when("between", () => thresholdUpper != null && value >= threshold && value <= thresholdUpper), - Match.when( - "not_between", - () => thresholdUpper != null && (value < threshold || value > thresholdUpper), - ), - Match.exhaustive, - ) - const makePersistenceError = (error: unknown) => { const cause = describeCause(error instanceof Error ? error.cause : error) return new AlertPersistenceError({ @@ -553,79 +505,26 @@ export class AlertsService extends Context.Service, reasonOverride?: string, - ): EvaluatedRule => { - const noDataBehavior = rule.compiledPlan.noDataBehavior - // Sample-weighted counts arrive fractional from the warehouse - // (`sum(SampleRate)`), and this flows into `last_sample_count`, an - // `integer` column — an unrounded value fails the insert outright. - const sampleCount = Math.round(obs.sampleCount) - const value = obs.hasData ? obs.value : noDataBehavior === "zero" ? 0 : null - - if (!obs.hasData && noDataBehavior === "skip") { - return { - status: "skipped", - value: null, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, + ): EvaluatedRule => + evaluateAlertObservation( + { comparator: rule.comparator, - reason: "No data in the selected window", - // Inert: `skipped` never resolves an incident, so this branch - // short-circuits before any status is derived from a synthesized value. - derivedFromNoData: false, - } - } - - if (sampleCount < rule.minimumSampleCount) { - return { - status: "skipped", - value, - sampleCount, threshold: rule.threshold, thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: `Sample count ${sampleCount} is below minimum ${rule.minimumSampleCount}`, - derivedFromNoData: false, - } - } - - if (value == null) { - return { - status: "skipped", - value: null, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: "Alert evaluation did not return a scalar value", - derivedFromNoData: false, - } - } - - return { - status: compareThreshold(value, rule.comparator, rule.threshold, rule.thresholdUpper) - ? "breached" - : "healthy", - value, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: - reasonOverride ?? + minimumSampleCount: rule.minimumSampleCount, + noDataBehavior: rule.compiledPlan.noDataBehavior, + }, + obs, + reasonOverride ?? `${rule.signalType} ${formatComparator(rule.comparator, rule.threshold, rule.thresholdUpper)}`, - // Only reachable with `noDataBehavior: "zero"` — the "skip" branch - // returned above. The comparison ran against a fabricated 0. - derivedFromNoData: !obs.hasData, - } - } + ) const buildDeliveryKey = ( incidentId: string, destinationId: string, eventType: AlertEventTypeValue, scheduledAt: number, - ) => [incidentId, destinationId, eventType, scheduledAt].join(":") + ) => makeAlertDeliveryKey(incidentId, destinationId, eventType, scheduledAt) const insertDeliveryEventRecord = ( db: DatabaseExecutor, @@ -811,9 +710,8 @@ export class AlertsService extends Context.Service= against *Required, so saturating keeps open/resolve behavior - // identical while letting steady-state ticks skip the state upsert above. - const consecutiveBreaches = - evaluation.status === "breached" - ? Math.min( - (state?.consecutiveBreaches ?? 0) + 1, - normalized.consecutiveBreachesRequired, - ) - : 0 - const consecutiveHealthy = - evaluation.status === "healthy" - ? Math.min( - (state?.consecutiveHealthy ?? 0) + 1, - normalized.consecutiveHealthyRequired, - ) - : 0 - + let lifecycle = planAlertLifecycle(lifecycleInput) + // Persist the counter/state decision before follow-up adapter work, as + // before extraction. A failed flap-history or liveness query must not + // discard an evaluation that already completed successfully. yield* upsertState({ - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, lastStatus: evaluation.status, lastValue: evaluation.value, lastSampleCount: evaluation.sampleCount, }) - if ( - evaluation.status === "breached" && - openIncident == null && - consecutiveBreaches >= normalized.consecutiveBreachesRequired - ) { - // Flap suppression: a metric oscillating around the threshold opens - // a fresh incident per flap, which would email an identical trigger - // notification every few minutes. If the previous incident for this - // (rule, group) was notified within the renotify interval, open the - // incident but skip the trigger notification and carry the prior - // lastNotifiedAt forward — the renotify gate then enforces one - // email per interval while the flapping persists. + // Ask the persistence adapter for flap history only when the pure core + // has decided that a new incident is otherwise ready to open. + if (lifecycle.transition === "opened") { const priorNotified = (yield* dbExecute((db) => db @@ -1728,9 +1615,46 @@ export class AlertsService extends Context.Service db.insert(alertIncidents).values(incident)) - if (flapSuppressedAt != null) { + if (lifecycle.notificationSuppression === "flapping") { yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe( Effect.annotateLogs({ ruleId: row.id, incidentId, groupKey, - priorNotifiedAt: flapSuppressedAt.toISOString(), + priorNotifiedAt: inheritedNotificationAt?.toISOString(), }), ) - } else { + } else if (lifecycle.eventType === "trigger") { yield* queueIncidentNotifications( row.orgId, normalized, incident, evaluation, - "trigger", + lifecycle.eventType, timestamp, ) } return { - transition: "opened" as const, + transition: lifecycle.transition, incidentId, openedIncidentId: incidentId, - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, } } - if (evaluation.status === "breached" && openIncident != null) { + if (lifecycle.transition === "continued" && openIncident != null) { const refreshedIncident = { ...openIncident, lastTriggeredAt: new Date(timestamp), @@ -1799,19 +1720,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -1821,72 +1729,32 @@ export class AlertsService extends Context.Service= normalized.consecutiveHealthyRequired - ) { - // A "healthy" synthesized from an empty window is a statement - // about missing data, not about a recovered system: with - // `noDataBehavior: "zero"` a total ingest outage compares as 0 < - // threshold and would resolve every incident it touches, paging - // out a wave of false all-clears. Believe it only once telemetry - // is provably still arriving. - if (evaluation.derivedFromNoData) { - const liveness = yield* telemetryStillFlowing( - row.orgId, - normalized, - openIncident.firstTriggeredAt.getTime(), - timestamp, - ) - if (!liveness.dataFlowing) { - yield* Effect.logWarning( - "Holding incident open: healthy evaluation came from missing telemetry", - ).pipe( - Effect.annotateLogs({ - orgId: row.orgId, - ruleId: row.id, - incidentId: openIncident.id, - groupKey, - livenessReason: liveness.reason, - observedCount: liveness.observedCount, - baselineCount: liveness.baselineCount, - }), - ) - return { - transition: "none" as const, - incidentId: carriedIncidentId, - openedIncidentId: null, - consecutiveBreaches, - consecutiveHealthy, - } - } - } - + if (lifecycle.transition === "resolved" && openIncident != null) { const resolvedIncident = { ...openIncident, status: "resolved" as const, @@ -1896,7 +1764,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -1910,14 +1777,7 @@ export class AlertsService extends Context.Service=4.0.0-beta.100 || >=4.0.0", }, }, + "packages/alerting-core": { + "name": "@maple/alerting-core", + "version": "0.0.0", + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/auth": { "name": "@maple/auth", "dependencies": { @@ -1519,6 +1529,8 @@ "@maple/alerting": ["@maple/alerting@workspace:apps/alerting"], + "@maple/alerting-core": ["@maple/alerting-core@workspace:packages/alerting-core"], + "@maple/api": ["@maple/api@workspace:apps/api"], "@maple/auth": ["@maple/auth@workspace:packages/auth"], diff --git a/packages/alerting-core/README.md b/packages/alerting-core/README.md new file mode 100644 index 000000000..a464cce3a --- /dev/null +++ b/packages/alerting-core/README.md @@ -0,0 +1,28 @@ +# `@maple/alerting-core` + +Host-neutral alert evaluation and incident-lifecycle semantics shared by Maple +deployment targets. + +The core is deliberately free of database, telemetry warehouse, scheduler, +network, and wall-clock dependencies. A host supplies observations and durable +state, calls the pure decision functions, then applies the returned transition +and delivery intent through its own adapters. + +Current hosted adapters live in `apps/api` and are scheduled by +`apps/alerting`. A Maple Local adapter can use the same core with chDB-backed +queries, Local durable state, an in-process scheduler, and its own outbound URL +policy without importing either hosted application. + +The boundary is: + +- query adapter -> `AlertObservation`; +- evaluation policy + observation -> `AlertEvaluation`; +- persistence snapshot + evaluation -> `AlertLifecyclePlan`; +- host persists the plan and sends its optional `eventType` through a delivery + adapter; +- delivery adapters share idempotency-key and bounded retry policy helpers; +- host clock supplies `nowMs`; the core never reads global time. + +Rule CRUD, storage schemas, scheduler claims, destination configuration, and +delivery transports remain host concerns. This keeps Local UI work optional: +the alert runtime can evaluate and deliver while no browser is open. diff --git a/packages/alerting-core/package.json b/packages/alerting-core/package.json new file mode 100644 index 000000000..0ce27ea84 --- /dev/null +++ b/packages/alerting-core/package.json @@ -0,0 +1,18 @@ +{ + "name": "@maple/alerting-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/alerting-core/src/index.test.ts b/packages/alerting-core/src/index.test.ts new file mode 100644 index 000000000..9ad7c26dc --- /dev/null +++ b/packages/alerting-core/src/index.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from "vitest" +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + type AlertEvaluation, +} from "./index" + +const breached: AlertEvaluation = { + status: "breached", + value: 11, + sampleCount: 5, + threshold: 10, + thresholdUpper: null, + comparator: "gt", + reason: "above threshold", + derivedFromNoData: false, +} + +const healthy: AlertEvaluation = { ...breached, status: "healthy", value: 9 } + +const policy = { + consecutiveBreachesRequired: 2, + consecutiveHealthyRequired: 2, + renotifyIntervalMinutes: 10, +} + +describe("evaluateAlertObservation", () => { + it("applies thresholds and rounds weighted sample counts", () => { + expect( + evaluateAlertObservation( + { + comparator: "between", + threshold: 10, + thresholdUpper: 20, + minimumSampleCount: 2, + noDataBehavior: "skip", + }, + { value: 15, sampleCount: 2.4, hasData: true }, + "inside range", + ), + ).toMatchObject({ status: "breached", sampleCount: 2, reason: "inside range" }) + }) + + it("marks a zero synthesized from no data so lifecycle resolution can fail closed", () => { + expect( + evaluateAlertObservation( + { + comparator: "gt", + threshold: 10, + thresholdUpper: null, + minimumSampleCount: 0, + noDataBehavior: "zero", + }, + { value: null, sampleCount: 0, hasData: false }, + "above threshold", + ), + ).toMatchObject({ status: "healthy", value: 0, derivedFromNoData: true }) + }) +}) + +describe("planAlertLifecycle", () => { + it("opens only after the configured breach count", () => { + const first = planAlertLifecycle({ + policy, + evaluation: breached, + state: null, + openIncident: null, + nowMs: 1_000, + }) + expect(first).toMatchObject({ transition: "none", state: { consecutiveBreaches: 1 } }) + + const second = planAlertLifecycle({ + policy, + evaluation: breached, + state: first.state, + openIncident: null, + nowMs: 2_000, + }) + expect(second).toMatchObject({ transition: "opened", eventType: "trigger" }) + }) + + it("suppresses a flapping trigger and its matching resolve", () => { + const opened = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 1, consecutiveHealthy: 0 }, + openIncident: null, + nowMs: 600_000, + previousNotificationAtMs: 300_000, + }) + expect(opened).toMatchObject({ + transition: "opened", + eventType: null, + notificationSuppression: "flapping", + inheritedNotificationAtMs: 300_000, + }) + + const resolved = planAlertLifecycle({ + policy, + evaluation: healthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 600_000, + lastNotifiedAtMs: opened.inheritedNotificationAtMs, + lastDeliveredEventType: null, + }, + nowMs: 700_000, + }) + expect(resolved).toMatchObject({ + transition: "resolved", + eventType: null, + notificationSuppression: "flap_resolution", + }) + }) + + it("advances the notification anchor when renotify becomes due", () => { + const plan = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 2, consecutiveHealthy: 0 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 1_000, + lastDeliveredEventType: "trigger", + }, + nowMs: 601_000, + }) + expect(plan).toMatchObject({ + transition: "continued", + eventType: "renotify", + advanceNotificationAnchor: true, + }) + }) + + it("holds a no-data recovery until the host proves telemetry liveness", () => { + const noDataHealthy = { ...healthy, derivedFromNoData: true } + const input = { + policy, + evaluation: noDataHealthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 0, + lastDeliveredEventType: "trigger" as const, + }, + nowMs: 1_000, + } + expect(planAlertLifecycle(input)).toMatchObject({ transition: "none", hold: "missing_telemetry" }) + expect(planAlertLifecycle({ ...input, allowNoDataResolution: true })).toMatchObject({ + transition: "resolved", + eventType: "resolve", + }) + }) +}) + +describe("interleaveAlertRulesByTenant", () => { + it("preserves each tenant's order while round-robining tenants", () => { + const rows = [ + { tenantId: "a", id: "a1" }, + { tenantId: "a", id: "a2" }, + { tenantId: "b", id: "b1" }, + { tenantId: "a", id: "a3" }, + { tenantId: "b", id: "b2" }, + ] + expect(interleaveAlertRulesByTenant(rows, (row) => row.tenantId).map(({ id }) => id)).toEqual([ + "a1", + "b1", + "a2", + "b2", + "a3", + ]) + }) +}) + +describe("delivery policy", () => { + it("builds stable idempotency keys", () => { + expect(makeAlertDeliveryKey("incident", "destination", "trigger", 42)).toBe( + "incident:destination:trigger:42", + ) + }) + + it("caps exponential retry delay and attempts", () => { + expect(alertDeliveryRetryDelayMs(1, 123)).toBe(60_123) + expect(alertDeliveryRetryDelayMs(5, 999)).toBe(900_999) + expect(canRetryAlertDelivery(4, true)).toBe(true) + expect(canRetryAlertDelivery(5, true)).toBe(false) + expect(canRetryAlertDelivery(1, false)).toBe(false) + }) +}) diff --git a/packages/alerting-core/src/index.ts b/packages/alerting-core/src/index.ts new file mode 100644 index 000000000..4130d25b6 --- /dev/null +++ b/packages/alerting-core/src/index.ts @@ -0,0 +1,321 @@ +export type AlertComparator = "gt" | "gte" | "lt" | "lte" | "eq" | "neq" | "between" | "not_between" + +export type AlertEvaluationStatus = "breached" | "healthy" | "skipped" + +export interface AlertObservation { + readonly value: number | null + readonly sampleCount: number + readonly hasData: boolean +} + +export interface AlertEvaluationPolicy { + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly minimumSampleCount: number + readonly noDataBehavior: "skip" | "zero" +} + +export interface AlertEvaluation { + readonly status: AlertEvaluationStatus + readonly value: number | null + readonly sampleCount: number + readonly threshold: number + readonly thresholdUpper: number | null + readonly comparator: AlertComparator + readonly reason: string + /** A healthy result derived from an empty window synthesized as zero. */ + readonly derivedFromNoData: boolean +} + +export const compareAlertThreshold = ( + value: number, + comparator: AlertComparator, + threshold: number, + thresholdUpper: number | null = null, +): boolean => { + switch (comparator) { + case "gt": + return value > threshold + case "gte": + return value >= threshold + case "lt": + return value < threshold + case "lte": + return value <= threshold + case "eq": + return value === threshold + case "neq": + return value !== threshold + case "between": + return thresholdUpper != null && value >= threshold && value <= thresholdUpper + case "not_between": + return thresholdUpper != null && (value < threshold || value > thresholdUpper) + } +} + +export const evaluateAlertObservation = ( + policy: AlertEvaluationPolicy, + observation: AlertObservation, + reason: string, +): AlertEvaluation => { + // Sample-weighted counts can be fractional while durable alert state commonly + // stores an integer. Normalize at the host-neutral boundary. + const sampleCount = Math.round(observation.sampleCount) + const value = observation.hasData ? observation.value : policy.noDataBehavior === "zero" ? 0 : null + + if (!observation.hasData && policy.noDataBehavior === "skip") { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "No data in the selected window", + derivedFromNoData: false, + } + } + + if (sampleCount < policy.minimumSampleCount) { + return { + status: "skipped", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: `Sample count ${sampleCount} is below minimum ${policy.minimumSampleCount}`, + derivedFromNoData: false, + } + } + + if (value == null) { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "Alert evaluation did not return a scalar value", + derivedFromNoData: false, + } + } + + return { + status: compareAlertThreshold(value, policy.comparator, policy.threshold, policy.thresholdUpper) + ? "breached" + : "healthy", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason, + derivedFromNoData: !observation.hasData, + } +} + +export interface AlertLifecyclePolicy { + readonly consecutiveBreachesRequired: number + readonly consecutiveHealthyRequired: number + readonly renotifyIntervalMinutes: number +} + +export interface AlertLifecycleState { + readonly consecutiveBreaches: number + readonly consecutiveHealthy: number +} + +export interface AlertLifecycleIncident { + readonly firstTriggeredAtMs: number + readonly lastNotifiedAtMs: number | null + readonly lastDeliveredEventType: AlertEventType | null +} + +export type AlertEventType = "trigger" | "resolve" | "renotify" | "test" +export type AlertIncidentTransition = "none" | "opened" | "continued" | "resolved" +export type AlertNotificationSuppression = "flapping" | "flap_resolution" | null +export type AlertLifecycleHold = "missing_telemetry" | null + +export interface AlertLifecycleInput { + readonly policy: AlertLifecyclePolicy + readonly evaluation: AlertEvaluation + readonly state: AlertLifecycleState | null + readonly openIncident: AlertLifecycleIncident | null + readonly nowMs: number + /** Most recent notification for a resolved incident with the same rule and group. */ + readonly previousNotificationAtMs?: number | null + /** Set only after the host's telemetry-query adapter proves data is still arriving. */ + readonly allowNoDataResolution?: boolean +} + +export interface AlertLifecyclePlan { + readonly state: AlertLifecycleState + readonly transition: AlertIncidentTransition + readonly eventType: AlertEventType | null + readonly notificationSuppression: AlertNotificationSuppression + readonly hold: AlertLifecycleHold + /** Notification anchor to copy to a newly opened, flap-suppressed incident. */ + readonly inheritedNotificationAtMs: number | null + /** Whether the host must advance lastNotifiedAt before queueing the event. */ + readonly advanceNotificationAnchor: boolean +} + +export interface AlertDeliveryRetryPolicy { + readonly maxAttempts: number + readonly baseDelayMs: number + readonly maxDelayMs: number +} + +export const DEFAULT_ALERT_DELIVERY_RETRY_POLICY: AlertDeliveryRetryPolicy = { + maxAttempts: 5, + baseDelayMs: 60_000, + maxDelayMs: 15 * 60_000, +} + +/** Stable idempotency key shared by every alert delivery adapter. */ +export const makeAlertDeliveryKey = ( + incidentId: string, + destinationId: string, + eventType: AlertEventType, + scheduledAtMs: number, +): string => [incidentId, destinationId, eventType, scheduledAtMs].join(":") + +export const canRetryAlertDelivery = ( + attemptNumber: number, + retryable: boolean, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): boolean => retryable && attemptNumber < policy.maxAttempts + +/** Exponential retry delay; the host supplies jitter from its own random source. */ +export const alertDeliveryRetryDelayMs = ( + attemptNumber: number, + jitterMs: number, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): number => { + const exponent = Math.max(0, attemptNumber - 1) + const base = Math.min(policy.baseDelayMs * Math.pow(2, exponent), policy.maxDelayMs) + return base + Math.max(0, Math.floor(jitterMs)) +} + +const noTransition = (state: AlertLifecycleState, hold: AlertLifecycleHold = null): AlertLifecyclePlan => ({ + state, + transition: "none", + eventType: null, + notificationSuppression: null, + hold, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, +}) + +/** + * Decide the next alert state and lifecycle intent without performing I/O. + * + * The caller owns persistence, incident identifiers, delivery, telemetry + * liveness checks, and time. This makes the same lifecycle semantics usable by + * the hosted PostgreSQL/Tinybird adapter and a future Maple Local adapter. + */ +export const planAlertLifecycle = (input: AlertLifecycleInput): AlertLifecyclePlan => { + const { evaluation, policy, openIncident, nowMs } = input + const previous = input.state ?? { consecutiveBreaches: 0, consecutiveHealthy: 0 } + + if (evaluation.status === "skipped") return noTransition(previous) + + const state: AlertLifecycleState = { + consecutiveBreaches: + evaluation.status === "breached" + ? Math.min(previous.consecutiveBreaches + 1, policy.consecutiveBreachesRequired) + : 0, + consecutiveHealthy: + evaluation.status === "healthy" + ? Math.min(previous.consecutiveHealthy + 1, policy.consecutiveHealthyRequired) + : 0, + } + + if ( + evaluation.status === "breached" && + openIncident == null && + state.consecutiveBreaches >= policy.consecutiveBreachesRequired + ) { + const previousNotificationAtMs = input.previousNotificationAtMs ?? null + const flapSuppressed = + previousNotificationAtMs != null && + previousNotificationAtMs >= nowMs - policy.renotifyIntervalMinutes * 60_000 + return { + state, + transition: "opened", + eventType: flapSuppressed ? null : "trigger", + notificationSuppression: flapSuppressed ? "flapping" : null, + hold: null, + inheritedNotificationAtMs: flapSuppressed ? previousNotificationAtMs : null, + advanceNotificationAnchor: false, + } + } + + if (evaluation.status === "breached" && openIncident != null) { + const renotifyDueAt = + (openIncident.lastNotifiedAtMs ?? openIncident.firstTriggeredAtMs) + + policy.renotifyIntervalMinutes * 60_000 + const renotifyDue = renotifyDueAt <= nowMs + return { + state, + transition: "continued", + eventType: renotifyDue ? "renotify" : null, + notificationSuppression: null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: renotifyDue, + } + } + + if ( + evaluation.status === "healthy" && + openIncident != null && + state.consecutiveHealthy >= policy.consecutiveHealthyRequired + ) { + if (evaluation.derivedFromNoData && input.allowNoDataResolution !== true) { + return noTransition(state, "missing_telemetry") + } + + const flapResolutionSuppressed = + openIncident.lastDeliveredEventType == null && openIncident.lastNotifiedAtMs != null + return { + state, + transition: "resolved", + eventType: flapResolutionSuppressed ? null : "resolve", + notificationSuppression: flapResolutionSuppressed ? "flap_resolution" : null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, + } + } + + return noTransition(state) +} + +/** Preserve per-tenant order while preventing one tenant from monopolizing a tick. */ +export const interleaveAlertRulesByTenant = ( + rows: ReadonlyArray, + tenantIdOf: (row: T) => string, +): ReadonlyArray => { + const queues = new Map() + for (const row of rows) { + const tenantId = tenantIdOf(row) + const queue = queues.get(tenantId) + if (queue) queue.push(row) + else queues.set(tenantId, [row]) + } + + const fair: T[] = [] + let index = 0 + while (fair.length < rows.length) { + for (const queue of queues.values()) { + const row = queue[index] + if (row !== undefined) fair.push(row) + } + index += 1 + } + return fair +} diff --git a/packages/alerting-core/tsconfig.json b/packages/alerting-core/tsconfig.json new file mode 100644 index 000000000..3d83a7d0c --- /dev/null +++ b/packages/alerting-core/tsconfig.json @@ -0,0 +1,17 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + } +} From 6b29b842674bfca2e5bf54edfafeab861c7c83f6 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 7 Aug 2026 20:45:29 -0400 Subject: [PATCH 02/12] feat(eventing): add typed signal projection architecture --- apps/api/package.json | 1 + .../src/planetscale-webhook-runtime.test.ts | 42 +- apps/api/src/planetscale-webhook-runtime.ts | 13 +- .../v1/planetscale-webhook.http.test.ts | 11 +- .../src/routes/v1/planetscale-webhook.http.ts | 29 +- .../alerts/AlertDestinationDelivery.ts | 24 +- apps/api/src/services/alerts/AlertsService.ts | 43 +- .../PlanetScaleWebhookQueue.test.ts | 26 +- .../planetscale/PlanetScaleWebhookQueue.ts | 2 + .../planetscale/webhook-events.test.ts | 58 ++ .../planetscale/webhook-events.ts | 159 +++- apps/cli/package.json | 1 + apps/cli/src/server/checkpoints.ts | 102 +- apps/cli/src/server/eventing/control-store.ts | 437 +++++++++ apps/cli/src/server/eventing/otlp.ts | 411 ++++++++ apps/cli/src/server/eventing/runtime.ts | 217 +++++ apps/cli/src/server/serve.ts | 166 +++- apps/cli/test/checkpoints.test.ts | 44 +- .../test/local-eventing-control-store.test.ts | 159 ++++ apps/cli/test/local-eventing-ingest.test.ts | 138 +++ apps/cli/test/local-eventing-runtime.test.ts | 203 ++++ apps/cli/test/server-network.test.ts | 3 +- bun.lock | 20 + docs/signal-to-event-projection.md | 881 ++++++++++++++++++ packages/alerting-core/README.md | 16 +- packages/alerting-core/package.json | 3 + packages/alerting-core/src/index.test.ts | 34 + packages/alerting-core/src/index.ts | 83 ++ packages/eventing-core/README.md | 22 + packages/eventing-core/fixtures/v1.json | 177 ++++ packages/eventing-core/package.json | 24 + .../schemas/cloud-event.v1.schema.json | 162 ++++ .../schemas/signal-projection.v1.schema.json | 349 +++++++ .../schemas/signal-scalar.v1.schema.json | 110 +++ .../eventing-core/scripts/generate-schemas.ts | 61 ++ packages/eventing-core/src/event.ts | 117 +++ packages/eventing-core/src/index.ts | 5 + packages/eventing-core/src/model.ts | 219 +++++ packages/eventing-core/src/predicate.test.ts | 148 +++ packages/eventing-core/src/predicate.ts | 341 +++++++ packages/eventing-core/src/registry.test.ts | 261 ++++++ packages/eventing-core/src/registry.ts | 184 ++++ packages/eventing-core/src/source.ts | 141 +++ packages/eventing-core/tsconfig.json | 23 + 44 files changed, 5581 insertions(+), 89 deletions(-) create mode 100644 apps/cli/src/server/eventing/control-store.ts create mode 100644 apps/cli/src/server/eventing/otlp.ts create mode 100644 apps/cli/src/server/eventing/runtime.ts create mode 100644 apps/cli/test/local-eventing-control-store.test.ts create mode 100644 apps/cli/test/local-eventing-ingest.test.ts create mode 100644 apps/cli/test/local-eventing-runtime.test.ts create mode 100644 docs/signal-to-event-projection.md create mode 100644 packages/eventing-core/README.md create mode 100644 packages/eventing-core/fixtures/v1.json create mode 100644 packages/eventing-core/package.json create mode 100644 packages/eventing-core/schemas/cloud-event.v1.schema.json create mode 100644 packages/eventing-core/schemas/signal-projection.v1.schema.json create mode 100644 packages/eventing-core/schemas/signal-scalar.v1.schema.json create mode 100644 packages/eventing-core/scripts/generate-schemas.ts create mode 100644 packages/eventing-core/src/event.ts create mode 100644 packages/eventing-core/src/index.ts create mode 100644 packages/eventing-core/src/model.ts create mode 100644 packages/eventing-core/src/predicate.test.ts create mode 100644 packages/eventing-core/src/predicate.ts create mode 100644 packages/eventing-core/src/registry.test.ts create mode 100644 packages/eventing-core/src/registry.ts create mode 100644 packages/eventing-core/src/source.ts create mode 100644 packages/eventing-core/tsconfig.json diff --git a/apps/api/package.json b/apps/api/package.json index 206780653..bc06507f6 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -41,6 +41,7 @@ "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", diff --git a/apps/api/src/planetscale-webhook-runtime.test.ts b/apps/api/src/planetscale-webhook-runtime.test.ts index 6c877a323..858b76567 100644 --- a/apps/api/src/planetscale-webhook-runtime.test.ts +++ b/apps/api/src/planetscale-webhook-runtime.test.ts @@ -1,27 +1,41 @@ import type { MessageBatch } from "@cloudflare/workers-types" import { afterEach, assert, describe, it } from "@effect/vitest" -import { Effect, Layer } from "effect" +import { OrgId } from "@maple/domain/http" +import { Effect, Layer, Schema } from "effect" import { Database, DatabaseError } from "@/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@/platform/test-pglite" import { processPlanetScaleWebhookBatch } from "./planetscale-webhook-runtime" +import { projectPlanetScaleWebhookEvent } from "./services/integrations/planetscale/webhook-events" import type { PlanetScaleWebhookJob } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) -const job: PlanetScaleWebhookJob = { - kind: "planetscale-webhook", - orgId: "org_1", - connectionId: "connection_1", - payload: { +const orgId = Schema.decodeUnknownSync(OrgId)("org_1") + +const makeJob = ( + payload: PlanetScaleWebhookJob["payload"] = { event: "branch.out_of_memory", organization: "acme", database: "shop", resource: { name: "main" }, }, +): PlanetScaleWebhookJob => ({ + kind: "planetscale-webhook", + orgId, + connectionId: "connection_1", + payload, receivedAt: 1_000, -} + event: projectPlanetScaleWebhookEvent({ + orgId, + connectionId: "connection_1", + payload, + receivedAt: 1_000, + }), +}) + +const job = makeJob() const makeBatch = (body: unknown) => { let acknowledged = false @@ -86,10 +100,7 @@ describe("PlanetScale webhook queue consumer", () => { it.effect("writes a lifecycle event to the timeline but not to the issue hub", () => { const testDb = createTestDb(trackedDbs) - const delivery = makeBatch({ - ...job, - payload: { ...job.payload, event: "branch.ready" }, - }) + const delivery = makeBatch(makeJob({ ...job.payload, event: "branch.ready" })) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) assert.isTrue(delivery.acknowledged()) @@ -138,14 +149,13 @@ describe("PlanetScale webhook queue consumer", () => { it.effect("carries the deploy-request number so redelivery dedupes", () => { const testDb = createTestDb(trackedDbs) - const delivery = makeBatch({ - ...job, - payload: { + const delivery = makeBatch( + makeJob({ ...job.payload, event: "deploy_request.schema_applied", resource: { number: 42 }, - }, - }) + }), + ) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) const event = yield* Effect.promise(() => diff --git a/apps/api/src/planetscale-webhook-runtime.ts b/apps/api/src/planetscale-webhook-runtime.ts index 71bc6c596..5a9e80fd2 100644 --- a/apps/api/src/planetscale-webhook-runtime.ts +++ b/apps/api/src/planetscale-webhook-runtime.ts @@ -54,9 +54,20 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => ), ), onSuccess: (job) => { - const classified = classifyPlanetScaleEvent(job.payload.event) + const event = job.event + const eventData = + typeof event.data === "object" && + event.data !== null && + !Array.isArray(event.data) + ? (event.data as { readonly [key: string]: unknown }) + : null + const eventName = + typeof eventData?.event === "string" ? eventData.event : job.payload.event + const classified = classifyPlanetScaleEvent(eventName) const annotateJob = Effect.annotateCurrentSpan({ orgId: job.orgId, + "maple.event.id": event.id, + "maple.event.type": event.type, "maple.planetscale.connection_id": job.connectionId, "maple.planetscale.webhook.event": job.payload.event, }) diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts index 3b22e91f4..3d5bd92ba 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts @@ -187,11 +187,13 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(jobs[0]?.orgId, "org_1") assert.strictEqual(jobs[0]?.connectionId, CONNECTION_ID) assert.strictEqual(jobs[0]?.payload.event, "branch.out_of_memory") + assert.strictEqual(jobs[0]?.event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(jobs[0]?.event.tenantid, "org_1") }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) - it.effect("enqueues lifecycle events too, and still drops genuinely unknown ones", () => { + it.effect("enqueues every verified factual event before downstream classification", () => { const testDb = createTestDb(trackedDbs) const jobs: PlanetScaleWebhookJob[] = [] return Effect.gen(function* () { @@ -260,15 +262,16 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(branchReady.status, 202) assert.strictEqual(jobs.length, 2) - // Forward-compatibility must not become "enqueue everything": an - // event neither side knows is acknowledged and dropped. + // Unknown provider facts also enter the typed event layer. The current + // issue/timeline consumer may ignore them, but other consumers can opt in. const unknown = yield* post({ event: "branch.some_future_event", organization: "acme", database: "shop", }) assert.strictEqual(unknown.status, 202) - assert.strictEqual(jobs.length, 2) + assert.strictEqual(jobs.length, 3) + assert.strictEqual(jobs[2]?.payload.event, "branch.some_future_event") }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.ts b/apps/api/src/routes/v1/planetscale-webhook.http.ts index a943d18a1..1336c59be 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.ts @@ -9,6 +9,7 @@ import { Env } from "@/platform/Env" import { classifyPlanetScaleEvent, decodePlanetScaleWebhookPayload, + projectPlanetScaleWebhookEvent, verifyPlanetScaleSignature, } from "@/services/integrations/planetscale/webhook-events" import { PlanetScaleWebhookQueue } from "@/services/integrations/planetscale/PlanetScaleWebhookQueue" @@ -166,17 +167,33 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => return textResponse("ok", 200) } - // Both issue-worthy and timeline-only events go through the queue: the - // durable retry is what makes a missed deploy marker recoverable. - if (classified.action === "issue" || classified.action === "timeline") { + // Every verified factual event is normalized and projected before the + // durable queue boundary. The queued CloudEvent is the stable contract; + // payload remains temporarily for parity with the existing consumers. + { const now = yield* Clock.currentTimeMillis + const orgId = decodeOrgIdSync(connection.orgId) + const event = yield* Effect.try({ + try: () => + projectPlanetScaleWebhookEvent({ + orgId, + connectionId, + payload, + receivedAt: now, + }), + catch: () => + new PlanetScaleWebhookUnavailable({ + body: "Webhook event projection unavailable", + }), + }) const enqueued = yield* webhookQueue .send({ kind: "planetscale-webhook", - orgId: decodeOrgIdSync(connection.orgId), + orgId, connectionId, payload, receivedAt: now, + event, }) .pipe( Effect.tapError((error) => @@ -201,10 +218,6 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => event: payload.event, }), ) - } else { - yield* Effect.logInfo("PlanetScale webhook lifecycle event acknowledged").pipe( - Effect.annotateLogs({ orgId: connection.orgId, event: payload.event }), - ) } yield* Effect.annotateCurrentSpan({ diff --git a/apps/api/src/services/alerts/AlertDestinationDelivery.ts b/apps/api/src/services/alerts/AlertDestinationDelivery.ts index 9e7b3ee08..17916469a 100644 --- a/apps/api/src/services/alerts/AlertDestinationDelivery.ts +++ b/apps/api/src/services/alerts/AlertDestinationDelivery.ts @@ -5,6 +5,7 @@ import { type AlertIncidentId, type AlertRuleId, } from "@maple/domain/http" +import { projectAlertLifecycleEvent } from "@maple/alerting-core" import type { AlertDestinationRow } from "@maple/db" import { Effect } from "effect" import { parseBase64Aes256GcmKey } from "@/platform/Crypto" @@ -127,8 +128,26 @@ export const makeAlertDestinationDelivery = (options: { { sendEmail, resolveSlackBotToken: options.resolveSlackBotToken }, ) - const buildPayload = (context: AlertDeliveryPayloadContext) => + const buildPayload = (context: AlertDeliveryPayloadContext, tenantId: string) => ({ + event: projectAlertLifecycleEvent({ + tenantId, + ruleId: context.ruleId, + ruleName: context.ruleName, + incidentId: context.incidentId, + eventType: context.eventType, + incidentStatus: context.incidentStatus, + groupKey: context.groupKey, + signalType: context.signalType, + severity: context.severity, + comparator: context.comparator, + threshold: context.threshold, + thresholdUpper: context.thresholdUpper, + windowMinutes: context.windowMinutes, + value: context.value, + sampleCount: context.sampleCount, + occurredAtMs: context.sentAtMs, + }), eventType: context.eventType, incidentId: context.incidentId, incidentStatus: context.incidentStatus, @@ -153,6 +172,7 @@ export const makeAlertDestinationDelivery = (options: { chatUrl: buildAlertChatUrl(options.appBaseUrl, context), sentAt: new Date(context.sentAtMs).toISOString(), }) satisfies { + readonly event: ReturnType readonly eventType: AlertDeliveryPayloadContext["eventType"] readonly incidentId: AlertIncidentId | null readonly incidentStatus: AlertDeliveryPayloadContext["incidentStatus"] @@ -177,7 +197,7 @@ export const makeAlertDestinationDelivery = (options: { secretConfig: enrichedSecret, ...context, } - const payload = buildPayload(fullContext) + const payload = buildPayload(fullContext, destinationRow.orgId) return yield* dispatchDelivery(fullContext, JSON.stringify(payload)) }) diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 1b382929b..2404ac4d6 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -659,26 +659,29 @@ export class AlertsService extends Context.Service [row.id, row])) - const payload = buildPayload({ - eventType, - incidentId: incident.id, - incidentStatus: decodeAlertIncidentStatusSync(incident.status), - dedupeKey: incident.dedupeKey, - ruleId: rule.id, - ruleName: rule.name, - groupKey: incident.groupKey, - signalType: rule.signalType, - severity: rule.severity, - comparator: rule.comparator, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - windowMinutes: rule.windowMinutes, - value: evaluation.value, - sampleCount: evaluation.sampleCount, - template: rule.notificationTemplate, - linkUrl: resolveNotificationLinkUrl(rule, incident.groupKey), - sentAtMs: scheduledAt, - }) + const payload = buildPayload( + { + eventType, + incidentId: incident.id, + incidentStatus: decodeAlertIncidentStatusSync(incident.status), + dedupeKey: incident.dedupeKey, + ruleId: rule.id, + ruleName: rule.name, + groupKey: incident.groupKey, + signalType: rule.signalType, + severity: rule.severity, + comparator: rule.comparator, + threshold: rule.threshold, + thresholdUpper: rule.thresholdUpper, + windowMinutes: rule.windowMinutes, + value: evaluation.value, + sampleCount: evaluation.sampleCount, + template: rule.notificationTemplate, + linkUrl: resolveNotificationLinkUrl(rule, incident.groupKey), + sentAtMs: scheduledAt, + }, + orgId, + ) yield* Effect.forEach( rule.destinationIds, diff --git a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts index 0bd3f5c2f..a32063655 100644 --- a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts +++ b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts @@ -1,19 +1,29 @@ import { assert, describe, it } from "@effect/vitest" +import { OrgId } from "@maple/domain/http" import { WorkerEnvironment } from "@maple/effect-cloudflare" -import { Effect, Layer } from "effect" +import { Effect, Layer, Schema } from "effect" +import { projectPlanetScaleWebhookEvent } from "./webhook-events" import { PlanetScaleWebhookQueue, type PlanetScaleWebhookJob } from "./PlanetScaleWebhookQueue" +const orgId = Schema.decodeUnknownSync(OrgId)("org_1") +const payload = { + event: "branch.anomaly", + organization: "acme", + database: "shop", + resource: { name: "main" }, +} const job: PlanetScaleWebhookJob = { kind: "planetscale-webhook", - orgId: "org_1", + orgId, connectionId: "connection_1", - payload: { - event: "branch.anomaly", - organization: "acme", - database: "shop", - resource: { name: "main" }, - }, + payload, receivedAt: 1_000, + event: projectPlanetScaleWebhookEvent({ + orgId, + connectionId: "connection_1", + payload, + receivedAt: 1_000, + }), } const provideQueue = (environment: Record) => diff --git a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts index bdf053340..5f6674ab1 100644 --- a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts +++ b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts @@ -1,6 +1,7 @@ import type { Queue } from "@cloudflare/workers-types" import { OrgId } from "@maple/domain/http" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { MapleCloudEventSchema } from "@maple/eventing-core" import { Context, Data, Effect, Layer, Schema } from "effect" import { PlanetScaleWebhookPayload } from "./webhook-events" @@ -12,6 +13,7 @@ export const PlanetScaleWebhookJob = Schema.Struct({ connectionId: Schema.String, payload: PlanetScaleWebhookPayload, receivedAt: Schema.Number, + event: MapleCloudEventSchema, }) export type PlanetScaleWebhookJob = Schema.Schema.Type diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts index eb4a0f1d7..1a1383a1d 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts @@ -10,6 +10,7 @@ import { deployRequestNumber, insertPlanetScaleEvent, planetScaleIssueFingerprint, + projectPlanetScaleWebhookEvent, truncateToSecond, upsertPlanetScaleIssue, verifyPlanetScaleSignature, @@ -47,6 +48,63 @@ describe("verifyPlanetScaleSignature", () => { }) describe("classifyPlanetScaleEvent", () => { + it("normalizes queued webhooks into deterministic common CloudEvents", () => { + const payload = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const input = { + orgId: "org_events", + connectionId: "connection-1", + payload, + receivedAt: 1_698_252_880_000, + } + const event = projectPlanetScaleWebhookEvent(input) + assert.deepStrictEqual(event, projectPlanetScaleWebhookEvent(input)) + assert.strictEqual(event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(event.tenantid, "org_events") + assert.strictEqual(event.subject, "planetscale-databases/main-db") + assert.strictEqual((event.data as { readonly event: string }).event, "branch.out_of_memory") + assert.throws( + () => projectPlanetScaleWebhookEvent({ ...input, receivedAt: Number.MAX_SAFE_INTEGER }), + /outside the supported date range/, + ) + }) + + it("keeps source-timestamp identities stable and receipt-time fallbacks payload-consistent", () => { + const timestamped = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const first = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_880_000, + }) + const redelivery = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_990_000, + }) + assert.strictEqual(first.id, redelivery.id) + assert.strictEqual(first.time, redelivery.time) + + const withoutTimestamp = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)({ + event: "branch.ready", + database: "main-db", + }) + const receivedFirst = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: withoutTimestamp, + receivedAt: 1_698_252_880_000, + }) + const receivedAgain = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: withoutTimestamp, + receivedAt: 1_698_252_990_000, + }) + assert.notStrictEqual(receivedFirst.id, receivedAgain.id) + assert.notStrictEqual(receivedFirst.time, receivedAgain.time) + }) + it("maps health events to issues and lifecycle events to timeline rows", () => { assert.strictEqual(classifyPlanetScaleEvent("branch.out_of_memory").action, "issue") assert.strictEqual(classifyPlanetScaleEvent("branch.anomaly").action, "issue") diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.ts b/apps/api/src/services/integrations/planetscale/webhook-events.ts index d63e173d6..dc8d0af63 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.ts @@ -1,4 +1,15 @@ -import { createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { + canonicalJson, + CompiledProjectionRegistry, + defineSignalFields, + ProjectorRegistry, + SignalSourceRegistry, + type JsonValue, + type MapleCloudEvent, + type SignalProjector, + type SignalSourceAdapter, +} from "@maple/eventing-core" import type { IssueSeverity, OrgId, WorkflowState } from "@maple/domain/http" import { ActorId, ErrorIssueEventId, ErrorIssueId } from "@maple/domain/primitives" import { @@ -49,6 +60,152 @@ export const decodePlanetScaleWebhookPayload = Schema.decodeUnknownEffect( Schema.fromJsonString(PlanetScaleWebhookPayload), ) +export interface PlanetScaleWebhookEventInput { + readonly orgId: string + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload + readonly receivedAt: number +} + +interface PlanetScaleWebhookAdapterInput { + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload +} + +interface PlanetScaleWebhookAdapterContext { + readonly tenantId: string + readonly acceptedAt: string +} + +const validDate = (epochMs: number, label: string): Date => { + if (!Number.isSafeInteger(epochMs) || epochMs < 0) + throw new Error(`${label} must be a non-negative epoch millisecond`) + const date = new Date(epochMs) + if (Number.isNaN(date.getTime())) throw new Error(`${label} is outside the supported date range`) + return date +} + +export const PLANETSCALE_WEBHOOK_ADAPTER: SignalSourceAdapter< + PlanetScaleWebhookAdapterInput, + PlanetScaleWebhookAdapterContext +> = { + definition: { + sourceKind: "planetscale.webhook", + fields: [ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "unavailable", + }, + ], + }, + normalize: ({ connectionId, payload }, context) => { + const observedAtDate = new Date(context.acceptedAt) + if (Number.isNaN(observedAtDate.getTime())) + throw new Error("PlanetScale receipt time is outside the supported date range") + const payloadJson = payload as unknown as JsonValue + const occurredAtMs = + payload.timestamp != null && payload.timestamp > 0 + ? Math.trunc(payload.timestamp * 1_000) + : observedAtDate.getTime() + const occurredAt = validDate(occurredAtMs, "PlanetScale event timestamp").toISOString() + const occurrenceId = `derived:sha256:${createHash("sha256") + .update(connectionId) + .update("\0") + .update(canonicalJson(payloadJson)) + .update("\0") + .update(occurredAt) + .digest("hex")}` + return [ + { + sourceKind: "planetscale.webhook", + source: `urn:maple:planetscale:${connectionId}`, + tenantId: context.tenantId, + occurrenceId, + identityQuality: "derived", + occurredAt, + observedAt: observedAtDate.toISOString(), + subject: + payload.database == null + ? `planetscale-connections/${connectionId}` + : `planetscale-databases/${payload.database}`, + fields: defineSignalFields([ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: payload.event }, + }, + ]), + data: { + connectionId, + event: payload.event, + organization: payload.organization ?? null, + database: payload.database ?? null, + resource: (payload.resource ?? null) as JsonValue, + }, + }, + ] + }, +} + +const PLANETSCALE_WEBHOOK_PROJECTOR: SignalProjector> = { + id: "planetscale.webhook", + version: 1, + sourceKinds: ["planetscale.webhook"], + outputType: "dev.maple.planetscale.webhook.received.v1", + dataSchema: "urn:maple:event-schema:planetscale-webhook:v1", + decodeConfig: (value) => { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + Object.keys(value).length > 0 + ) + throw new Error("PlanetScale webhook projector config must be empty") + return {} + }, + project: (signal) => ({ data: signal.data as JsonValue }), +} + +const PLANETSCALE_SOURCES = new SignalSourceRegistry().register(PLANETSCALE_WEBHOOK_ADAPTER.definition) +const PLANETSCALE_PROJECTORS = new ProjectorRegistry().register(PLANETSCALE_WEBHOOK_PROJECTOR) + +/** Normalize and project one verified, durably queued PlanetScale webhook through the common layer. */ +export const projectPlanetScaleWebhookEvent = (input: PlanetScaleWebhookEventInput): MapleCloudEvent => { + const observedAt = validDate(input.receivedAt, "PlanetScale receipt time").toISOString() + const [signal] = PLANETSCALE_WEBHOOK_ADAPTER.normalize( + { connectionId: input.connectionId, payload: input.payload }, + { tenantId: input.orgId, acceptedAt: observedAt }, + ) + if (!signal) throw new Error("PlanetScale webhook adapter produced no signal") + const registry = CompiledProjectionRegistry.compile( + [ + { + id: "planetscale-webhook", + revision: 1, + enabled: true, + tenantId: input.orgId, + sourceKind: "planetscale.webhook", + selector: { + op: "exists", + field: { namespace: "signal", key: "event.name", type: "string" }, + }, + projector: { id: "planetscale.webhook", version: 1, config: {} }, + activeFrom: observedAt, + }, + ], + PLANETSCALE_SOURCES, + PLANETSCALE_PROJECTORS, + ) + const result = registry.evaluate(signal) + if (result.failures.length > 0) throw new Error(result.failures[0]!.message) + if (result.events.length !== 1) throw new Error("PlanetScale webhook projection produced no event") + return result.events[0]! +} + +// --------------------------------------------------------------------------- +// Classification +// --------------------------------------------------------------------------- /** Where an event belongs on the timeline. Mirrored by the web vocabulary table. */ export type PlanetScaleEventCategory = "deploy_request" | "branch" | "database" | "cluster" | "keyspace" diff --git a/apps/cli/package.json b/apps/cli/package.json index d7ed8f30b..0cc45719c 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -14,6 +14,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1" diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index cfa8a7e92..894f3279c 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto" +import { createHash, randomUUID } from "node:crypto" import { spawnSync } from "node:child_process" import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { cp, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises" @@ -18,6 +18,11 @@ import { syncDirectory, syncTree, } from "./durable-files" +import { + eventingControlSnapshotPath, + LocalEventingControlStore, + type EventingControlSnapshotValidation, +} from "./eventing/control-store" import { CURRENT_LOCAL_SCHEMA, SCHEMA_FINGERPRINT } from "./schema-identity" import schemaSql from "./schema/local-schema.sql" with { type: "text" } import { @@ -28,12 +33,12 @@ import { } from "./store-version" const STATE_FORMAT_VERSION = 1 -const MANIFEST_FORMAT_VERSION = 1 +const MANIFEST_FORMAT_VERSION = 2 const OPERATION_FORMAT_VERSION = 1 const RESTORE_TRANSACTION_FORMAT_VERSION = 1 const RESET_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i -const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "status", "store", "tmp"]) +const RESETTABLE_LIVE_ENTRIES = new Set(["control", "data", "metadata", "status", "store", "tmp"]) export const CHECKPOINT_REOPEN_PROBE_ENV = "MAPLE_INTERNAL_CHECKPOINT_REOPEN_DATA_DIR" const CheckpointUuid = Schema.String.check(Schema.isPattern(CHECKPOINT_ID)) @@ -83,8 +88,7 @@ const CheckpointValidationSchema = Schema.Struct({ export type CheckpointValidation = Schema.Schema.Type -const CheckpointManifestSchema = Schema.Struct({ - formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), +const CheckpointManifestFields = { checkpointId: CheckpointId, operationId: CheckpointOperationId, mapleVersion: Schema.String, @@ -95,8 +99,28 @@ const CheckpointManifestSchema = Schema.Struct({ backupRelativePath: Schema.String, backupBytes: NonNegativeInt, validation: CheckpointValidationSchema, +} as const + +const EventingControlSnapshotValidationSchema = Schema.Struct({ + schemaVersion: NonNegativeInt, + projectionRevisions: NonNegativeInt, + projectionFailures: NonNegativeInt, + stagedEvents: NonNegativeInt, + readyEvents: NonNegativeInt, }) +const CheckpointManifestSchema = Schema.Union([ + Schema.Struct({ formatVersion: Schema.Literal(1), ...CheckpointManifestFields }), + Schema.Struct({ + formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), + ...CheckpointManifestFields, + controlRelativePath: Schema.String, + controlBytes: NonNegativeInt, + controlSha256: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + controlValidation: EventingControlSnapshotValidationSchema, + }), +]) + export type CheckpointManifest = Schema.Schema.Type const CheckpointStateSchema = Schema.Struct({ @@ -124,7 +148,7 @@ const RestoreTransactionPhase = Schema.Literals([ "markers-committed", ]) const ResetTransactionPhase = Schema.Literals(["intent", "live-cleared", "markers-cleared"]) -const ResetTarget = Schema.Literals(["data", "metadata", "status", "store", "tmp"]) +const ResetTarget = Schema.Literals(["control", "data", "metadata", "status", "store", "tmp"]) const CheckpointOperationSchema = Schema.Struct({ formatVersion: Schema.Literal(OPERATION_FORMAT_VERSION), @@ -313,9 +337,23 @@ const snapshotManifestPath = (dataDir: string, checkpointId: CheckpointId): stri const snapshotBackupDir = (dataDir: string, checkpointId: CheckpointId): string => join(checkpointSnapshotDir(dataDir, checkpointId), "backup") const snapshotBackupRelativePath = (checkpointId: CheckpointId): string => `snapshots/${checkpointId}/backup` +const snapshotControlRelativePath = (checkpointId: CheckpointId): string => + `snapshots/${checkpointId}/control.sqlite` const snapshotBackupSqlPath = (checkpointId: CheckpointId): string => `backups/${snapshotBackupRelativePath(checkpointId)}` +const sha256File = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex") + +const controlValidationMatches = ( + left: EventingControlSnapshotValidation, + right: EventingControlSnapshotValidation, +): boolean => + left.schemaVersion === right.schemaVersion && + left.projectionRevisions === right.projectionRevisions && + left.projectionFailures === right.projectionFailures && + left.stagedEvents === right.stagedEvents && + left.readyEvents === right.readyEvents + const assertContained = (root: string, candidate: string, label: string): string => { const absoluteRoot = resolve(root) const absoluteCandidate = resolve(candidate) @@ -653,6 +691,12 @@ export const parseCheckpointManifest = ( if (manifest.backupRelativePath !== snapshotBackupRelativePath(manifest.checkpointId)) { throw new Error("checkpoint backup path does not match its immutable ID") } + if ( + manifest.formatVersion === MANIFEST_FORMAT_VERSION && + manifest.controlRelativePath !== snapshotControlRelativePath(manifest.checkpointId) + ) { + throw new Error("checkpoint control-store path does not match its immutable ID") + } if (manifest.chdbVersion !== CHDB_VERSION) { throw new Error( `checkpoint chDB version mismatch (checkpoint: ${manifest.chdbVersion}; build: ${CHDB_VERSION})`, @@ -763,6 +807,24 @@ const resolveCheckpointById = async ( `checkpoint backup size mismatch (manifest: ${manifest.backupBytes}; actual: ${actualBackupBytes})`, ) } + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + if (manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await assertNoSymlink(snapshotsRoot, controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlBytes = (await stat(controlPath)).size + if (controlBytes !== manifest.controlBytes) + throw new Error( + `checkpoint control-store size mismatch (manifest: ${manifest.controlBytes}; actual: ${controlBytes})`, + ) + const controlSha256 = sha256File(controlPath) + if (controlSha256 !== manifest.controlSha256) + throw new Error("checkpoint control-store digest mismatch") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + if (!controlValidationMatches(manifest.controlValidation, controlValidation)) + throw new Error("checkpoint control-store validation does not match its manifest") + } else if (existsSync(controlPath)) { + throw new Error("legacy checkpoint contains an unsigned eventing control snapshot") + } return { checkpointId, snapshotDir, @@ -804,6 +866,15 @@ const restoreResolvedInto = async ( `RESTORE DATABASE default FROM Disk('src', '${resolvedCheckpoint.backupSqlPath}') ` + "SETTINGS allow_different_database_def=1", ) + if (resolvedCheckpoint.manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await LocalEventingControlStore.restoreSnapshot( + join(resolvedCheckpoint.snapshotDir, "control.sqlite"), + targetDataDir, + ) + } else { + const controlStore = await LocalEventingControlStore.open(targetDataDir) + controlStore.close() + } return { db, validation: validateRestoredDatabase(db) } } catch (error) { db?.close() @@ -1505,10 +1576,14 @@ export const createCheckpoint = Effect.fn("CheckpointService.create")(function* const { oldState, snapshot, startedAt } = prepared let { operation } = prepared await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) + const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) operation = { ...operation, phase: "backup-complete" } await writeOperation(options.dataDir, operation, options.faults) const provisionalManifest: CheckpointManifest = { - formatVersion: 1, + formatVersion: MANIFEST_FORMAT_VERSION, checkpointId, operationId, mapleVersion: MAPLE_VERSION, @@ -1518,6 +1593,10 @@ export const createCheckpoint = Effect.fn("CheckpointService.create")(function* sourceDataDir: resolve(options.dataDir), backupRelativePath: snapshotBackupRelativePath(checkpointId), backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), + controlRelativePath: snapshotControlRelativePath(checkpointId), + controlBytes: (await stat(controlPath)).size, + controlSha256: sha256File(controlPath), + controlValidation, validation: { validatedAt: startedAt, traces: 0, @@ -1688,7 +1767,7 @@ const beginResetTransactionUnlocked = async ( const entries = await readdir(live, { withFileTypes: true }) for (const entry of entries) { if (entry.name === "backups") continue - if (!RESETTABLE_CHDB_ENTRIES.has(entry.name)) { + if (!RESETTABLE_LIVE_ENTRIES.has(entry.name)) { unknown.push(join(live, entry.name)) continue } @@ -1936,9 +2015,10 @@ export const reconcileCheckpointRecovery = Effect.fn("CheckpointService.reconcil }) /** - * Explicitly remove the live chDB store while preserving the checkpoint - * registry below `/backups`. The maintenance lock serializes this - * destructive operation with checkpoint, restore, and archive work. + * Explicitly remove the live chDB and eventing control stores while preserving + * the checkpoint registry below `/backups`. The maintenance lock + * serializes this destructive operation with checkpoint, restore, and archive + * work. */ export const resetLiveStorePreservingCheckpoints = Effect.fn("CheckpointService.reset")(function* ( dataDir: string, diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts new file mode 100644 index 000000000..df8780ef7 --- /dev/null +++ b/apps/cli/src/server/eventing/control-store.ts @@ -0,0 +1,437 @@ +import { constants as sqliteConstants, Database } from "bun:sqlite" +import { chmodSync, existsSync, lstatSync, readFileSync } from "node:fs" +import { join, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { + canonicalJson, + isJsonValue, + MapleCloudEventSchema, + SignalProjectionSpecSchema, + type MapleCloudEvent, + type JsonValue, + type ProjectionFailure, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { durableWrite, ensurePrivateDirectory } from "../durable-files" + +const CONTROL_SCHEMA_VERSION = 1 +const CONTROL_DIRECTORY = "control" +const CONTROL_DATABASE = "eventing.sqlite" +const MAX_EVENT_BYTES = 256 * 1024 +const MAX_FAILURES_PER_TENANT = 10_000 + +export const eventingControlDirectory = (dataDir: string): string => join(resolve(dataDir), CONTROL_DIRECTORY) +export const eventingControlPath = (dataDir: string): string => + join(eventingControlDirectory(dataDir), CONTROL_DATABASE) +export const eventingControlSnapshotPath = (dataDir: string, checkpointId: string): string => + join(resolve(dataDir), "backups", "snapshots", checkpointId, "control.sqlite") + +const CREATE_SCHEMA = ` +CREATE TABLE projection_revisions ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (revision > 0), + enabled INTEGER NOT NULL CHECK (enabled IN (0, 1)), + spec_json TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (tenant_id, projection_id, revision) +) STRICT; + +CREATE TABLE active_projections ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (tenant_id, projection_id), + FOREIGN KEY (tenant_id, projection_id, revision) + REFERENCES projection_revisions (tenant_id, projection_id, revision) + ON DELETE RESTRICT +) STRICT; + +CREATE TABLE outbox_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + state TEXT NOT NULL CHECK (state IN ('staged', 'ready')), + event_json TEXT NOT NULL, + staged_at TEXT NOT NULL, + ready_at TEXT +) STRICT; + +CREATE INDEX outbox_events_ready_sequence + ON outbox_events (state, sequence); + +CREATE TABLE projection_failures ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + occurrence_id TEXT, + message TEXT NOT NULL, + created_at TEXT NOT NULL +) STRICT; + +CREATE UNIQUE INDEX projection_failures_occurrence + ON projection_failures (tenant_id, projection_id, projection_revision, occurrence_id) + WHERE occurrence_id IS NOT NULL; + +PRAGMA user_version = 1; +` + +interface UserVersionRow { + readonly user_version: number | bigint +} + +interface RevisionRow { + readonly revision: number | bigint | null +} + +interface ProjectionJsonRow { + readonly spec_json: string +} + +interface EventRow { + readonly event_id: string + readonly event_json: string + readonly state: "staged" | "ready" +} + +interface EventJsonRow { + readonly event_json: string +} + +interface CountRow { + readonly count: number | bigint +} + +interface QuickCheckRow { + readonly quick_check: string +} + +export interface StageEventsResult { + readonly inserted: number + readonly deduplicated: number + readonly eventIds: readonly string[] +} + +export interface EventingControlSnapshotValidation { + readonly schemaVersion: number + readonly projectionRevisions: number + readonly projectionFailures: number + readonly stagedEvents: number + readonly readyEvents: number +} + +const asNumber = (value: number | bigint): number => { + const number = Number(value) + if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) + return number +} + +const decodeProjection = (json: string): SignalProjectionSpec => + Schema.decodeUnknownSync(SignalProjectionSpecSchema)(JSON.parse(json) as unknown) + +const decodeEvent = (json: string): MapleCloudEvent => { + const value = Schema.decodeUnknownSync(MapleCloudEventSchema)(JSON.parse(json) as unknown) + if (!isJsonValue(value.data)) throw new Error("stored CloudEvent data is not finite JSON") + return value as MapleCloudEvent +} + +const assertRealDatabaseFile = (path: string): void => { + let info + try { + info = lstatSync(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + if (info.isSymbolicLink() || !info.isFile()) + throw new Error(`eventing control database is not a real file: ${path}`) +} + +const configure = (db: Database): void => { + db.exec("PRAGMA foreign_keys = ON") + db.exec("PRAGMA trusted_schema = OFF") + db.exec("PRAGMA busy_timeout = 5000") +} + +const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation => { + const quick = db.query("PRAGMA quick_check").get() + if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + const schemaVersion = asNumber(version.user_version) + if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + const count = (where: string): number => { + const row = db.query(`SELECT count(*) AS count FROM outbox_events ${where}`).get() + if (!row) throw new Error("eventing control count query returned no row") + return asNumber(row.count) + } + const revisions = db.query("SELECT count(*) AS count FROM projection_revisions").get() + if (!revisions) throw new Error("eventing projection count query returned no row") + const failures = db.query("SELECT count(*) AS count FROM projection_failures").get() + if (!failures) throw new Error("eventing projection-failure count query returned no row") + return { + schemaVersion, + projectionRevisions: asNumber(revisions.count), + projectionFailures: asNumber(failures.count), + stagedEvents: count("WHERE state = 'staged'"), + readyEvents: count("WHERE state = 'ready'"), + } +} + +export class LocalEventingControlStore { + readonly #db: Database + readonly path: string + + private constructor(path: string, db: Database) { + this.path = path + this.#db = db + } + + static async open(dataDir: string): Promise { + const directory = eventingControlDirectory(dataDir) + await ensurePrivateDirectory(directory) + const path = eventingControlPath(dataDir) + assertRealDatabaseFile(path) + const db = new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) + try { + configure(db) + db.exec("PRAGMA journal_mode = WAL") + db.exec("PRAGMA synchronous = FULL") + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + const schemaVersion = asNumber(version.user_version) + if (schemaVersion === 0) db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() + else if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + chmodSync(path, 0o600) + validateOpenDatabase(db) + return new LocalEventingControlStore(path, db) + } catch (error) { + db.close() + throw error + } + } + + close(): void { + this.#db.exec("PRAGMA wal_checkpoint(TRUNCATE)") + this.#db.close(true) + } + + saveProjection(spec: SignalProjectionSpec, createdAt = new Date().toISOString()): void { + const decoded = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(spec) + if (!isJsonValue(decoded as unknown)) throw new Error("projection spec must be finite JSON") + const specJson = canonicalJson(decoded as unknown as JsonValue) + this.#db + .transaction(() => { + const existing = this.#db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(decoded.tenantId, decoded.id, decoded.revision) + if (existing) { + if (existing.spec_json !== specJson) + throw new Error( + `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + } else { + const latest = this.#db + .query( + "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const expected = latest?.revision == null ? 1 : asNumber(latest.revision) + 1 + if (decoded.revision !== expected) + throw new Error( + `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + this.#db.run( + "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + decoded.tenantId, + decoded.id, + decoded.revision, + decoded.enabled ? 1 : 0, + specJson, + createdAt, + ], + ) + } + + if (decoded.enabled) + this.#db.run( + "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", + [decoded.tenantId, decoded.id, decoded.revision], + ) + else + this.#db.run("DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", [ + decoded.tenantId, + decoded.id, + ]) + }) + .immediate() + } + + loadEnabledProjections(tenantId: string): readonly SignalProjectionSpec[] { + return this.#db + .query( + `SELECT r.spec_json + FROM active_projections a + JOIN projection_revisions r + ON r.tenant_id = a.tenant_id + AND r.projection_id = a.projection_id + AND r.revision = a.revision + WHERE a.tenant_id = ? + ORDER BY a.projection_id`, + ) + .all(tenantId) + .map(({ spec_json }) => decodeProjection(spec_json)) + } + + stageEvents(events: readonly MapleCloudEvent[], stagedAt = new Date().toISOString()): StageEventsResult { + let inserted = 0 + let deduplicated = 0 + const eventIds: string[] = [] + this.#db + .transaction(() => { + for (const candidate of events) { + const event = Schema.decodeUnknownSync(MapleCloudEventSchema)(candidate) + if (!isJsonValue(event as unknown)) throw new Error("CloudEvent must be finite JSON") + const eventJson = canonicalJson(event as unknown as JsonValue) + if (Buffer.byteLength(eventJson, "utf8") > MAX_EVENT_BYTES) + throw new Error(`CloudEvent exceeds ${MAX_EVENT_BYTES} UTF-8 bytes`) + const existing = this.#db + .query( + "SELECT event_id, event_json, state FROM outbox_events WHERE event_id = ?", + ) + .get(event.id) + if (existing) { + if (existing.event_json !== eventJson) + throw new Error(`event ID collision with different payload: ${event.id}`) + deduplicated += 1 + } else { + this.#db.run( + "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, state, event_json, staged_at) VALUES (?, ?, ?, ?, 'staged', ?, ?)", + [ + event.id, + event.tenantid, + event.projectionid, + event.projectionrevision, + eventJson, + stagedAt, + ], + ) + inserted += 1 + } + eventIds.push(event.id) + } + }) + .immediate() + return { inserted, deduplicated, eventIds } + } + + markReady(eventIds: readonly string[], readyAt = new Date().toISOString()): void { + this.#db + .transaction(() => { + for (const eventId of eventIds) { + const row = this.#db + .query, [string]>( + "SELECT state FROM outbox_events WHERE event_id = ?", + ) + .get(eventId) + if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) + if (row.state === "ready") continue + this.#db.run( + "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", + [readyAt, eventId], + ) + } + }) + .immediate() + } + + listReady(limit = 100): readonly MapleCloudEvent[] { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new Error("ready-event limit must be between 1 and 1000") + return this.#db + .query( + "SELECT event_json FROM outbox_events WHERE state = 'ready' ORDER BY sequence LIMIT ?", + ) + .all(limit) + .map(({ event_json }) => decodeEvent(event_json)) + } + + listStaged(limit = 100): readonly MapleCloudEvent[] { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new Error("staged-event limit must be between 1 and 1000") + return this.#db + .query( + "SELECT event_json FROM outbox_events WHERE state = 'staged' ORDER BY sequence LIMIT ?", + ) + .all(limit) + .map(({ event_json }) => decodeEvent(event_json)) + } + + recordProjectionFailures( + tenantId: string, + failures: readonly ProjectionFailure[], + createdAt = new Date().toISOString(), + ): void { + this.#db + .transaction(() => { + for (const failure of failures) + this.#db.run( + "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + tenantId, + failure.projectionId, + failure.projectionRevision, + failure.occurrenceId, + failure.message.slice(0, 4_096), + createdAt, + ], + ) + this.#db.run( + "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", + [tenantId, tenantId, MAX_FAILURES_PER_TENANT], + ) + }) + .immediate() + } + + validate(): EventingControlSnapshotValidation { + return validateOpenDatabase(this.#db) + } + + async backupTo(path: string): Promise { + const bytes = this.#db.serialize() + await durableWrite(path, bytes) + return LocalEventingControlStore.validateSnapshot(path) + } + + static validateSnapshot(path: string): EventingControlSnapshotValidation { + assertRealDatabaseFile(path) + if (!existsSync(path)) throw new Error(`eventing control snapshot is missing: ${path}`) + const uri = `${pathToFileURL(path).href}?immutable=1` + const db = new Database(uri, sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI) + try { + configure(db) + return validateOpenDatabase(db) + } finally { + db.close(true) + } + } + + static async restoreSnapshot(snapshotPath: string, dataDir: string): Promise { + LocalEventingControlStore.validateSnapshot(snapshotPath) + await durableWrite(eventingControlPath(dataDir), readFileSync(snapshotPath)) + } +} diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts new file mode 100644 index 000000000..b93b4659c --- /dev/null +++ b/apps/cli/src/server/eventing/otlp.ts @@ -0,0 +1,411 @@ +import { createHash } from "node:crypto" +import { + canonicalJson, + defineSignalFields, + type JsonValue, + type NormalizedSignal, + type SignalFieldCatalogEntry, + type SignalScalar, + type SignalSourceAdapter, + type SignalSourceDefinition, +} from "@maple/eventing-core" +import { OtlpFieldError, spanIdHex, traceIdHex } from "../otlp/encode" + +interface AnyValue { + readonly stringValue?: string + readonly boolValue?: boolean + readonly intValue?: string | number + readonly doubleValue?: number + readonly bytesValue?: string + readonly arrayValue?: { readonly values?: readonly AnyValue[] } + readonly kvlistValue?: { readonly values?: readonly KeyValue[] } +} + +interface KeyValue { + readonly key?: string + readonly value?: AnyValue +} + +interface OtlpLogsRequest { + readonly resourceLogs?: readonly { + readonly resource?: { readonly attributes?: readonly KeyValue[] } + readonly scopeLogs?: readonly { + readonly scope?: { + readonly name?: string + readonly version?: string + readonly attributes?: readonly KeyValue[] + } + readonly logRecords?: readonly { + readonly timeUnixNano?: string | number + readonly observedTimeUnixNano?: string | number + readonly severityNumber?: number + readonly severityText?: string + readonly eventName?: string + readonly body?: AnyValue + readonly attributes?: readonly KeyValue[] + readonly traceId?: string + readonly spanId?: string + }[] + }[] + }[] +} + +const MAX_ATTRIBUTES = 256 +const MAX_STRING_BYTES = 16 * 1024 +const MAX_DATA_BYTES = 256 * 1024 +const MAX_VALUE_DEPTH = 8 +const MAX_VALUE_NODES = 1_024 +const SENSITIVE_KEY = + /(?:^|[._-])(authorization|cookie|password|passwd|secret|token|api[._-]?key)(?:$|[._-])/i + +const allOperators = ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"] as const +const equalityOperators = ["exists", "eq", "neq", "contains", "in"] as const + +const catalog = ( + key: string, + type: SignalScalar["type"], + operators: SignalFieldCatalogEntry["operators"] = allOperators, +): SignalFieldCatalogEntry => ({ + field: { namespace: "signal", key, type }, + operators, + sensitivity: "public", + replay: "exact", +}) + +export const OTLP_LOG_SOURCE: SignalSourceDefinition = { + sourceKind: "otel.log", + fields: [ + catalog("event.name", "string", equalityOperators), + catalog("severity.number", "int64"), + catalog("severity.text", "string", equalityOperators), + catalog("trace.id", "string", equalityOperators), + catalog("span.id", "string", equalityOperators), + catalog("time", "timestamp"), + catalog("observed_time", "timestamp"), + ], + openFields: [ + { + namespace: "resource", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "scope", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "body", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + ], +} + +interface ValueBudget { + nodes: number +} + +const assertStringBound = (value: string, label: string): string => { + if (Buffer.byteLength(value, "utf8") > MAX_STRING_BYTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes`) + return value +} + +const int64 = (value: string | number, label: string): string => { + if (typeof value === "number" && !Number.isSafeInteger(value)) + throw new OtlpFieldError( + `${label} must encode int64 as a decimal string when outside safe integer range`, + ) + const decimal = String(value) + if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) throw new OtlpFieldError(`${label} is not an int64`) + const parsed = BigInt(decimal) + if (parsed < -(1n << 63n) || parsed > (1n << 63n) - 1n) + throw new OtlpFieldError(`${label} is outside the int64 range`) + return decimal +} + +const anyValueScalar = (value: AnyValue | undefined, label: string): SignalScalar | null => { + if (!value) return null + if (value.stringValue !== undefined) + return { type: "string", value: assertStringBound(value.stringValue, label) } + if (value.boolValue !== undefined) return { type: "boolean", value: value.boolValue } + if (value.intValue !== undefined) return { type: "int64", value: int64(value.intValue, label) } + if (value.doubleValue !== undefined) { + if (!Number.isFinite(value.doubleValue)) throw new OtlpFieldError(`${label} must be finite`) + return { type: "float64", value: value.doubleValue } + } + return null +} + +const anyValueJson = ( + value: AnyValue | undefined, + label: string, + depth = 0, + budget: ValueBudget = { nodes: 0 }, +): JsonValue | null => { + budget.nodes += 1 + if (budget.nodes > MAX_VALUE_NODES) throw new OtlpFieldError(`${label} exceeds value node limit`) + if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError(`${label} exceeds value depth limit`) + const scalar = anyValueScalar(value, label) + if (scalar) return scalar.value + if (!value) return null + if (value.bytesValue !== undefined) return assertStringBound(value.bytesValue, `${label}.bytesValue`) + if (value.arrayValue !== undefined) + return (value.arrayValue.values ?? []).map((item, index) => + anyValueJson(item, `${label}[${index}]`, depth + 1, budget), + ) + if (value.kvlistValue !== undefined) { + const output: Record = {} + for (const [index, entry] of (value.kvlistValue.values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}.key[${index}]`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + output[key] = anyValueJson(entry.value, `${label}.${key}`, depth + 1, budget) + } + return output + } + return null +} + +interface NormalizedAttributes { + readonly scalars: ReadonlyArray<{ readonly key: string; readonly value: SignalScalar }> + readonly data: Readonly> +} + +const attributes = (values: readonly KeyValue[] | undefined, label: string): NormalizedAttributes => { + if ((values?.length ?? 0) > MAX_ATTRIBUTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_ATTRIBUTES} attributes`) + const scalars = new Map() + const data: Record = {} + for (const [index, entry] of (values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}[${index}].key`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + const scalar = anyValueScalar(entry.value, `${label}.${key}`) + if (scalar) scalars.set(key, scalar) + data[key] = anyValueJson(entry.value, `${label}.${key}`) + } + return { scalars: [...scalars].map(([key, value]) => ({ key, value })), data } +} + +const epochNanos = (value: string | number | undefined): bigint | null => { + if (value === undefined || value === "" || value === 0 || value === "0") return null + try { + const parsed = BigInt(value) + return parsed >= 0 ? parsed : null + } catch { + return null + } +} + +const nanosToTimestamp = (nanos: bigint): string => { + const seconds = nanos / 1_000_000_000n + const fraction = nanos % 1_000_000_000n + const milliseconds = Number(seconds) * 1_000 + const date = new Date(milliseconds) + if (!Number.isFinite(milliseconds) || Number.isNaN(date.getTime())) + throw new OtlpFieldError("OTLP timestamp is outside the supported date range") + return `${date.toISOString().slice(0, 19)}.${fraction.toString().padStart(9, "0")}Z` +} + +const stringAttribute = (attrs: NormalizedAttributes, key: string): string | null => { + const scalar = attrs.scalars.find((entry) => entry.key === key)?.value + return scalar?.type === "string" ? scalar.value : null +} + +const boundedIdentity = (value: string, prefix: string): string => + value.length <= 256 + ? value + : `${prefix}:sha256:${createHash("sha256").update(value, "utf8").digest("hex")}` + +const sourceUri = (resource: NormalizedAttributes, record: NormalizedAttributes): string => { + const explicit = stringAttribute(record, "event.source") ?? stringAttribute(record, "cloudevents.source") + if (explicit) return boundedIdentity(assertStringBound(explicit, "event source"), "urn:maple:source") + const service = stringAttribute(resource, "service.name") + const source = service + ? `urn:maple:source:otel:${encodeURIComponent(service)}` + : "urn:maple:source:otel:local" + return boundedIdentity(source, "urn:maple:source") +} + +const sourceOccurrenceId = (record: NormalizedAttributes): string | null => { + const value = + stringAttribute(record, "event.id") ?? + stringAttribute(record, "cloudevents.id") ?? + stringAttribute(record, "gitlab.event.id") + return value === null ? null : boundedIdentity(value, "source") +} + +const derivedOccurrenceId = (input: JsonValue): string => + `derived:sha256:${createHash("sha256").update(canonicalJson(input)).digest("hex")}` + +export const normalizeOtlpLogs = ( + request: unknown, + acceptedAt = new Date().toISOString(), + tenantId = "local", +): readonly NormalizedSignal[] => { + const input = (request ?? {}) as OtlpLogsRequest + const signals: NormalizedSignal[] = [] + for (const resourceLogs of input.resourceLogs ?? []) { + const resource = attributes(resourceLogs.resource?.attributes, "resource.attributes") + for (const scopeLogs of resourceLogs.scopeLogs ?? []) { + const scope = attributes(scopeLogs.scope?.attributes, "scope.attributes") + for (const log of scopeLogs.logRecords ?? []) { + const record = attributes(log.attributes, "log.attributes") + const occurredNanos = epochNanos(log.timeUnixNano) ?? epochNanos(log.observedTimeUnixNano) + const observedNanos = epochNanos(log.observedTimeUnixNano) + const occurredAt = occurredNanos ? nanosToTimestamp(occurredNanos) : acceptedAt + const sourceObservedAt = observedNanos ? nanosToTimestamp(observedNanos) : acceptedAt + const bodyScalar = anyValueScalar(log.body, "log.body") + const traceId = traceIdHex(log.traceId, "logRecord.traceId") + const spanId = spanIdHex(log.spanId, "logRecord.spanId") + const data: JsonValue = { + resource: resource.data, + scope: { + name: assertStringBound(scopeLogs.scope?.name ?? "", "scope.name"), + version: assertStringBound(scopeLogs.scope?.version ?? "", "scope.version"), + attributes: scope.data, + }, + record: { + eventName: assertStringBound(log.eventName ?? "", "log.eventName"), + severityNumber: log.severityNumber ?? 0, + severityText: assertStringBound(log.severityText ?? "", "log.severityText"), + traceId, + spanId, + body: anyValueJson(log.body, "log.body"), + attributes: record.data, + }, + } + if (Buffer.byteLength(canonicalJson(data), "utf8") > MAX_DATA_BYTES) + throw new OtlpFieldError(`normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes`) + const source = sourceUri(resource, record) + const occurrenceId = sourceOccurrenceId(record) + const subject = + stringAttribute(record, "event.subject") ?? stringAttribute(record, "cloudevents.subject") + signals.push({ + sourceKind: "otel.log", + source, + tenantId, + occurrenceId: + occurrenceId ?? + derivedOccurrenceId({ source, occurredAt, signalKind: "otel.log", data }), + identityQuality: occurrenceId ? "source" : "derived", + occurredAt, + observedAt: acceptedAt, + subject, + fields: defineSignalFields([ + ...(log.eventName + ? [ + { + field: { + namespace: "signal" as const, + key: "event.name", + type: "string" as const, + }, + value: { type: "string" as const, value: log.eventName }, + }, + ] + : []), + { + field: { namespace: "signal", key: "severity.number", type: "int64" }, + value: { + type: "int64", + value: int64(log.severityNumber ?? 0, "severity.number"), + }, + }, + ...(log.severityText + ? [ + { + field: { + namespace: "signal" as const, + key: "severity.text", + type: "string" as const, + }, + value: { type: "string" as const, value: log.severityText }, + }, + ] + : []), + ...(traceId + ? [ + { + field: { + namespace: "signal" as const, + key: "trace.id", + type: "string" as const, + }, + value: { type: "string" as const, value: traceId }, + }, + ] + : []), + ...(spanId + ? [ + { + field: { + namespace: "signal" as const, + key: "span.id", + type: "string" as const, + }, + value: { type: "string" as const, value: spanId }, + }, + ] + : []), + { + field: { namespace: "signal", key: "time", type: "timestamp" }, + value: { type: "timestamp", value: occurredAt }, + }, + { + field: { namespace: "signal", key: "observed_time", type: "timestamp" }, + value: { type: "timestamp", value: sourceObservedAt }, + }, + ...resource.scalars.map(({ key, value }) => ({ + field: { namespace: "resource" as const, key, type: value.type }, + value, + })), + ...scope.scalars.map(({ key, value }) => ({ + field: { namespace: "scope" as const, key, type: value.type }, + value, + })), + ...record.scalars.map(({ key, value }) => ({ + field: { namespace: "attribute" as const, key, type: value.type }, + value, + })), + ...(bodyScalar + ? [ + { + field: { + namespace: "body" as const, + key: "value", + type: bodyScalar.type, + }, + value: bodyScalar, + }, + ] + : []), + ]), + data, + }) + } + } + } + return signals +} + +export const OTLP_LOG_ADAPTER: SignalSourceAdapter< + unknown, + { readonly acceptedAt: string; readonly tenantId: string } +> = { + definition: OTLP_LOG_SOURCE, + normalize: (raw, context) => normalizeOtlpLogs(raw, context.acceptedAt, context.tenantId), +} diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts new file mode 100644 index 000000000..2655c5865 --- /dev/null +++ b/apps/cli/src/server/eventing/runtime.ts @@ -0,0 +1,217 @@ +import { + CompiledProjectionRegistry, + ProjectorRegistry, + SignalSourceRegistry, + fieldKey, + isJsonValue, + SignalProjectionSpecSchema, + type MapleCloudEvent, + type NormalizedSignal, + type ProjectionFailure, + type SignalProjectionSpec, + type SignalScalar, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { LocalEventingControlStore } from "./control-store" +import { OTLP_LOG_ADAPTER } from "./otlp" + +const TENANT_ID = "local" + +interface GitLabIssueProjectorConfig { + readonly includeBody: boolean +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const gitlabProjectorConfig = (value: unknown): GitLabIssueProjectorConfig => { + if (!isRecord(value)) throw new Error("gitlab.issue.created projector config must be an object") + const keys = Object.keys(value) + if (keys.some((key) => key !== "includeBody")) + throw new Error("gitlab.issue.created projector config contains an unknown field") + if (value.includeBody !== undefined && typeof value.includeBody !== "boolean") + throw new Error("gitlab.issue.created includeBody must be boolean") + return { includeBody: value.includeBody === true } +} + +const field = (signal: NormalizedSignal, namespace: "resource" | "attribute", key: string) => + signal.fields.get(fieldKey({ namespace, key })) + +const scalarString = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "string") throw new Error(`GitLab issue event ${label} must be a string`) + return value.value +} + +const scalarInt64 = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "int64") throw new Error(`GitLab issue event ${label} must be an int64`) + return value.value +} + +const gitlabIssueProjector = { + id: "gitlab.issue.created", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.created.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-created:v1", + decodeConfig: gitlabProjectorConfig, + project: (signal: NormalizedSignal, config: GitLabIssueProjectorConfig) => { + const projectId = scalarInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") + const projectPath = scalarString( + field(signal, "attribute", "gitlab.project.path"), + "gitlab.project.path", + true, + )! + const issueId = scalarInt64(field(signal, "attribute", "gitlab.issue.id"), "gitlab.issue.id") + const issueIid = scalarInt64( + field(signal, "attribute", "gitlab.issue.iid"), + "gitlab.issue.iid", + true, + )! + const title = scalarString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") + const url = scalarString(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") + const actorId = scalarInt64(field(signal, "attribute", "gitlab.user.id"), "gitlab.user.id") + const actorUsername = scalarString( + field(signal, "attribute", "gitlab.user.username"), + "gitlab.user.username", + ) + const serviceName = scalarString(field(signal, "resource", "service.name"), "service.name") + const candidateBody = + isRecord(signal.data) && isRecord(signal.data.record) ? signal.data.record.body : undefined + const body = isJsonValue(candidateBody) ? candidateBody : undefined + return { + subject: `${projectPath}/issues/${issueIid}`, + data: { + project: { + ...(projectId === undefined ? {} : { id: projectId }), + path: projectPath, + }, + issue: { + ...(issueId === undefined ? {} : { id: issueId }), + iid: issueIid, + ...(title === undefined ? {} : { title }), + ...(url === undefined ? {} : { url }), + }, + ...(actorId === undefined && actorUsername === undefined + ? {} + : { + actor: { + ...(actorId === undefined ? {} : { id: actorId }), + ...(actorUsername === undefined ? {} : { username: actorUsername }), + }, + }), + ...(serviceName === undefined ? {} : { serviceName }), + ...(config.includeBody && body !== undefined ? { body } : {}), + }, + } + }, +} as const + +export interface LocalProjectionEvaluation { + readonly events: readonly MapleCloudEvent[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +const emptyEvaluation = (): LocalProjectionEvaluation => ({ + events: [], + failures: [], + typeMismatchFields: [], +}) + +export class LocalEventingRuntime { + readonly #store: LocalEventingControlStore + readonly #sources: SignalSourceRegistry + readonly #projectors: ProjectorRegistry + #compiled: CompiledProjectionRegistry + #activeSourceKinds = new Set() + + constructor(store: LocalEventingControlStore) { + this.#store = store + this.#sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) + this.#projectors = new ProjectorRegistry().register(gitlabIssueProjector) + const specs = store.loadEnabledProjections(TENANT_ID) + this.#compiled = CompiledProjectionRegistry.compile(specs, this.#sources, this.#projectors) + this.#activeSourceKinds = new Set(specs.map(({ sourceKind }) => sourceKind)) + } + + hasActiveSource(sourceKind: string): boolean { + return this.#activeSourceKinds.has(sourceKind) + } + + activate(candidate: unknown): void { + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + if (spec.tenantId !== TENANT_ID) + throw new Error(`Maple Local only accepts projections for tenant ${TENANT_ID}`) + const active = this.#store + .loadEnabledProjections(TENANT_ID) + .filter((candidate) => candidate.id !== spec.id) + const next = spec.enabled ? [...active, spec] : active + const compiled = CompiledProjectionRegistry.compile(next, this.#sources, this.#projectors) + this.#store.saveProjection(spec) + this.#compiled = compiled + this.#activeSourceKinds = new Set(next.map(({ sourceKind }) => sourceKind)) + } + + listActive(): readonly SignalProjectionSpec[] { + return this.#store.loadEnabledProjections(TENANT_ID) + } + + evaluateOtlp( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay: (rangeDate: string) => boolean = () => false, + ): LocalProjectionEvaluation { + const sourceKind = signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" + if (!this.hasActiveSource(sourceKind)) return emptyEvaluation() + const acceptedAt = new Date().toISOString() + const normalized = ( + signal === "logs" ? OTLP_LOG_ADAPTER.normalize(decoded, { acceptedAt, tenantId: TENANT_ID }) : [] + ).filter((occurrence) => !isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) + const snapshot = this.#compiled + const events: MapleCloudEvent[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + for (const occurrence of normalized) { + const result = snapshot.evaluate(occurrence) + events.push(...result.events) + failures.push(...result.failures) + for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) + } + return { events, failures, typeMismatchFields: [...typeMismatchFields] } + } + + persistFailures(failures: readonly ProjectionFailure[]): void { + if (failures.length > 0) this.#store.recordProjectionFailures(TENANT_ID, failures) + } + + stage(events: readonly MapleCloudEvent[]) { + return this.#store.stageEvents(events) + } + + markReady(eventIds: readonly string[]): void { + this.#store.markReady(eventIds) + } + + listReady(limit?: number): readonly MapleCloudEvent[] { + return this.#store.listReady(limit) + } + + listStaged(limit?: number): readonly MapleCloudEvent[] { + return this.#store.listStaged(limit) + } + + health() { + return { + activeProjections: this.listActive().length, + ...this.#store.validate(), + } + } +} diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 24163767b..6d06ed0c9 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -17,6 +17,8 @@ import { rawTelemetryTtlStatements, } from "./chdb" import { buildInsertStatements } from "./inserts" +import { eventingControlSnapshotPath, LocalEventingControlStore } from "./eventing/control-store" +import { LocalEventingRuntime } from "./eventing/runtime" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { decodeLogsRequest, @@ -98,7 +100,7 @@ export const corsHeadersForAllowedOrigin = ( ? { "access-control-allow-origin": origin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": "content-type, content-encoding", + "access-control-allow-headers": "content-type, content-encoding, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", } @@ -166,6 +168,7 @@ interface IngestResult { async function ingest( db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise { @@ -195,6 +198,17 @@ async function ingest( requestBytes, } } + let evaluation: ReturnType + try { + evaluation = eventing.evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let batches: EncodedBatch[] try { batches = encodeFor(signal, decoded) @@ -209,6 +223,18 @@ async function ingest( requestBytes, } } + let stagedEventIds: readonly string[] = [] + try { + eventing.persistFailures(evaluation.failures) + if (evaluation.events.length > 0) stagedEventIds = eventing.stage(evaluation.events).eventIds + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let rejected = 0 batches = batches.map((batch) => { const filtered = authority.filterBatch(batch.datasource, batch.ndjson) @@ -231,6 +257,15 @@ async function ingest( accepted += statement.rowCount } } + try { + if (stagedEventIds.length > 0) eventing.markReady(stagedEventIds) + } catch (error) { + return { + response: text(`event outbox readiness ${signal}: ${(error as Error).message}`, 503), + accepted, + requestBytes, + } + } const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" if (contentType.includes("json")) { const rejectedField = @@ -390,6 +425,7 @@ const ingestSpan = ( runSpan: SpanRunner, db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise => @@ -397,7 +433,7 @@ const ingestSpan = ( recoverResponse( Effect.gen(function* () { const { response, accepted, requestBytes } = yield* Effect.promise(() => - ingest(db, authority, signal, req), + ingest(db, authority, eventing, signal, req), ) yield* Effect.annotateCurrentSpan({ "http.request.body.size": requestBytes, @@ -535,7 +571,13 @@ const handleRetirement = async ( const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i /** Typed, authenticated replacement for sending BACKUP through /local/query. */ -const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Promise => { +const handleCheckpointBackup = async ( + db: Chdb, + controlStore: LocalEventingControlStore, + dataDir: string, + token: string, + req: Request, +): Promise => { if (!maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token"))) return text("maintenance authorization required", 403) let body: unknown @@ -550,10 +592,10 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr return text("invalid checkpoint fields", 400) if (!CHECKPOINT_ID.test(record.checkpointId)) return text("invalid checkpoint ID", 400) try { - db.exec( - `BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${record.checkpointId.toLowerCase()}/backup')`, - ) - return json({ checkpointId: record.checkpointId.toLowerCase() }) + const checkpointId = record.checkpointId.toLowerCase() + const control = await controlStore.backupTo(eventingControlSnapshotPath(dataDir, checkpointId)) + db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) + return json({ checkpointId, control }) } catch (error) { return text( `checkpoint backup failed: ${error instanceof Error ? error.message : String(error)}`, @@ -562,6 +604,60 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr } } +const eventingAuthorized = (token: string, req: Request): Response | null => + maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token")) + ? null + : text("maintenance authorization required", 403) + +const handleProjectionActivation = async ( + eventing: LocalEventingRuntime, + token: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await req.json() + } catch { + return text("invalid JSON body", 400) + } + try { + eventing.activate(body) + return json({ active: eventing.listActive() }) + } catch (error) { + return text( + `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, + 400, + ) + } +} + +const handleEventingRead = ( + eventing: LocalEventingRuntime, + token: string, + req: Request, + url: URL, +): Response => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + if (url.pathname === "/local/eventing/health") return json(eventing.health()) + if (url.pathname === "/local/eventing/projections") return json(eventing.listActive()) + if (url.pathname === "/local/eventing/outbox") { + const rawLimit = url.searchParams.get("limit") + const limit = rawLimit === null ? 100 : Number(rawLimit) + const state = url.searchParams.get("state") ?? "ready" + try { + if (state === "ready") return json(eventing.listReady(limit)) + if (state === "staged") return json(eventing.listStaged(limit)) + return text("outbox state must be ready or staged", 400) + } catch (error) { + return text(error instanceof Error ? error.message : String(error), 400) + } + } + return text("not found", 404) +} + /** The `Bun.serve` fetch handler, closed over the chDB connection. Each ingest * and query request is run through `runSpan` so it leaves a trace; `/health` * and `OPTIONS` are skipped (loop-prevention convention — no health-check noise). */ @@ -573,6 +669,8 @@ const makeFetch = authority: RetiredDayAuthority, gate: RequestQuiescenceGate, maintenanceToken: string, + controlStore: LocalEventingControlStore, + eventing: LocalEventingRuntime, ) => async (req: Request): Promise => { const url = new URL(req.url) @@ -586,18 +684,34 @@ const makeFetch = if (url.pathname === "/health") return respond(text("OK")) if (req.method === "POST") { if (url.pathname === "/v1/traces") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "traces", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "traces", req)), + ) if (url.pathname === "/v1/logs") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "logs", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "logs", req)), + ) if (url.pathname === "/v1/metrics") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "metrics", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "metrics", req)), + ) if (url.pathname === "/local/query") return respond(await admitted(gate, () => querySpan(runSpan, db, authority, req))) if (url.pathname === "/local/checkpoint/backup") - return respond(await admitted(gate, () => handleCheckpointBackup(db, maintenanceToken, req))) + return respond( + await gate.exclusive(() => + handleCheckpointBackup(db, controlStore, options.dataDir, maintenanceToken, req), + ), + ) + if (url.pathname === "/local/eventing/projections") + return respond( + await gate.exclusive(() => handleProjectionActivation(eventing, maintenanceToken, req)), + ) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } + if (req.method === "GET" && url.pathname.startsWith("/local/eventing/")) + return respond(handleEventingRead(eventing, maintenanceToken, req, url)) if (req.method === "GET" && options.assets) return respond(serveAsset(options.assets, url.pathname)) return respond(text("not found", 404)) } @@ -633,6 +747,23 @@ export const startServer = ( configFile: options.configFile, rawTelemetryRetentionDays: retention.effective, }) + const controlStore = yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => LocalEventingControlStore.open(options.dataDir), + catch: (error) => + new ChdbError({ + message: `failed to open local eventing control store: ${error instanceof Error ? error.message : String(error)}`, + }), + }), + (store) => Effect.sync(() => store.close()), + ) + const eventing = yield* Effect.try({ + try: () => new LocalEventingRuntime(controlStore), + catch: (error) => + new ChdbError({ + message: `failed to compile local event projections: ${error instanceof Error ? error.message : String(error)}`, + }), + }) // `CREATE ... IF NOT EXISTS` does not repair a table whose physical // definition was altered out of band. Inspect the opened store before the // listener is bound; a mismatch fails startup rather than allowing new @@ -707,7 +838,16 @@ export const startServer = ( Bun.serve({ port: options.port, hostname: options.hostname, - fetch: makeFetch(db, options, runSpan, authority, gate, maintenanceToken), + fetch: makeFetch( + db, + options, + runSpan, + authority, + gate, + maintenanceToken, + controlStore, + eventing, + ), }), catch: (error) => new ServerBindError({ @@ -721,4 +861,4 @@ export const startServer = ( return { port: server.port ?? options.port } }) -export const __testables = { recordServerResponse } +export const __testables = { handleEventingRead, ingest, recordServerResponse } diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 720403aa1..946cd24bd 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,4 +1,5 @@ import { describe, it } from "@effect/vitest" +import { createHash } from "node:crypto" import { Effect, Exit, Option } from "effect" import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" import { @@ -54,6 +55,7 @@ import { import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { eventingControlSnapshotPath, LocalEventingControlStore } from "../src/server/eventing/control-store" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) @@ -306,6 +308,41 @@ describe("checkpoint IDs and strict parsers", () => { }) describe("checkpoint state resolution", () => { + it("binds a version-2 checkpoint to its eventing control snapshot", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const operationId = newCheckpointOperationId() + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "backup") + + const store = await LocalEventingControlStore.open(dataDir) + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + const controlValidation = await store.backupTo(controlPath) + store.close() + const controlBytes = readFileSync(controlPath) + writeFileSync( + join(snapshot, "manifest.json"), + `${JSON.stringify({ + ...manifest(checkpointId, operationId, dataDir), + formatVersion: 2, + backupBytes: 6, + controlRelativePath: `snapshots/${checkpointId}/control.sqlite`, + controlBytes: controlBytes.byteLength, + controlSha256: createHash("sha256").update(controlBytes).digest("hex"), + controlValidation, + })}\n`, + ) + writeState(dataDir, checkpointId) + strictEqual((await resolveCheckpoint(dataDir)).manifest.formatVersion, 2) + + const corrupted = Buffer.from(controlBytes) + corrupted[corrupted.length - 1] ^= 1 + writeFileSync(controlPath, corrupted) + await rejects(resolveCheckpoint(dataDir), /digest mismatch|quick_check failed/) + }) + }) + it("resolves immutable current, previous, and explicit IDs", async () => { await withDataDir(async (dataDir) => { const current = newCheckpointId() @@ -666,9 +703,11 @@ describe("live-store reset safety", () => { writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) mkdirSync(join(dataDir, "store"), { recursive: true }) + mkdirSync(join(dataDir, "control"), { recursive: true }) mkdirSync(join(dataDir, "metadata"), { recursive: true }) mkdirSync(join(dataDir, "tmp"), { recursive: true }) writeFileSync(join(dataDir, "store", "part.bin"), "live") + writeFileSync(join(dataDir, "control", "eventing.sqlite"), "live") writeFileSync(join(dataDir, "metadata", "table.sql"), "live") writeFileSync(join(dataDir, "status"), "live") writeFileSync(join(dataDir, "tmp", "scratch.bin"), "live") @@ -680,6 +719,7 @@ describe("live-store reset safety", () => { strictEqual((await readCheckpointState(dataDir)).current, checkpointId) ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) ok(!existsSync(join(dataDir, "store"))) + ok(!existsSync(join(dataDir, "control"))) ok(!existsSync(join(dataDir, "metadata"))) ok(!existsSync(join(dataDir, "status"))) ok(!existsSync(join(dataDir, "tmp"))) @@ -735,7 +775,7 @@ describe("live-store reset safety", () => { const checkpointId = newCheckpointId() writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) - for (const entry of ["data", "metadata", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "store", "tmp"]) { mkdirSync(join(dataDir, entry), { recursive: true }) writeFileSync(join(dataDir, entry, "live.bin"), "live") } @@ -757,7 +797,7 @@ describe("live-store reset safety", () => { ) await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) - for (const entry of ["data", "metadata", "status", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "status", "store", "tmp"]) { ok(!existsSync(join(dataDir, entry)), `${boundary}: ${entry}`) } strictEqual((await readCheckpointState(dataDir)).current, checkpointId) diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts new file mode 100644 index 000000000..2f592e866 --- /dev/null +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -0,0 +1,159 @@ +import { deepStrictEqual, rejects, strictEqual, throws } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import type { MapleCloudEvent, SignalProjectionSpec } from "@maple/eventing-core" +import { eventingControlPath, LocalEventingControlStore } from "../src/server/eventing/control-store" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-control-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 1, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + projector: { id: "gitlab.issue", version: 1, config: { includeTitle: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const event = (overrides: Partial = {}): MapleCloudEvent => ({ + specversion: "1.0", + id: "sha256:061c0b5d99b92ef65ab8813c6d84988e4b1582e705e0077c952e62a0e84b6b08", + source: "urn:maple:source:otel:local", + type: "dev.maple.gitlab.issue.created.v1", + subject: "project/example/issues/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:gitlab-issue:v1", + tenantid: "tenant-a", + projectionid: "gitlab-issue-created", + projectionrevision: 1, + projectorid: "gitlab.issue", + projectorversion: 1, + data: { iid: 42, title: "Example" }, + ...overrides, +}) + +describe("LocalEventingControlStore", () => { + it("stores immutable sequential revisions and only loads the active revision", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + throws( + () => + store.saveProjection( + projection({ projector: { id: "changed", version: 1, config: {} } }), + ), + /immutable/, + ) + throws(() => store.saveProjection(projection({ revision: 3 })), /must be 2/) + + store.saveProjection(projection({ revision: 2, enabled: false })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), []) + store.saveProjection(projection({ revision: 3 })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) + deepStrictEqual(store.validate(), { + schemaVersion: 1, + projectionRevisions: 3, + projectionFailures: 0, + stagedEvents: 0, + readyEvents: 0, + }) + } finally { + store.close() + } + })) + + it("deduplicates staged events, rejects collisions, and preserves ready order", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + deepStrictEqual(store.stageEvents([event(), event()]), { + inserted: 1, + deduplicated: 1, + eventIds: [event().id, event().id], + }) + throws(() => store.stageEvents([event({ data: { iid: 43 } })]), /collision/) + throws(() => store.markReady(["unknown"]), /unknown event/) + store.markReady([event().id]) + store.markReady([event().id]) + deepStrictEqual(store.listStaged(), []) + deepStrictEqual(store.listReady(), [event()]) + } finally { + store.close() + } + })) + + it("survives restart and round-trips through a validated standalone snapshot", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.saveProjection(projection()) + store.stageEvents([event()]) + store.markReady([event().id]) + store.recordProjectionFailures("tenant-a", [ + { + projectionId: "gitlab-issue-created", + projectionRevision: 1, + occurrenceId: "issue-42", + message: "test failure", + }, + ]) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(store.listReady(), [event()]) + const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") + const validation = await store.backupTo(snapshot) + deepStrictEqual(validation, { + schemaVersion: 1, + projectionRevisions: 1, + projectionFailures: 1, + stagedEvents: 0, + readyEvents: 1, + }) + store.close() + + const restored = join(dataDir, "restored") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + deepStrictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(restored)), + validation, + ) + const restoredStore = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(restoredStore.listReady(), [event()]) + } finally { + restoredStore.close() + } + })) + + it("refuses a symlink in place of the database", async () => + withDataDir(async (dataDir) => { + const controlPath = eventingControlPath(dataDir) + mkdirSync(join(dataDir, "control"), { recursive: true }) + symlinkSync(join(dataDir, "target.sqlite"), controlPath) + await rejects(() => LocalEventingControlStore.open(dataDir), /not a real file/) + strictEqual(controlPath.endsWith("control/eventing.sqlite"), true) + })) +}) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts new file mode 100644 index 000000000..a0e4b347e --- /dev/null +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -0,0 +1,138 @@ +import { deepStrictEqual, strictEqual } from "node:assert" +import { describe, it } from "vitest" +import { __testables } from "../src/server/serve" + +describe("Local eventing ingest seam", () => { + it("requires maintenance authorization and exposes staged records only when requested", async () => { + const eventing = { + health: () => ({ activeProjections: 1 }), + listActive: () => [], + listReady: () => [{ id: "ready" }], + listStaged: () => [{ id: "staged" }], + } + const unauthorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), + new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + ) + strictEqual(unauthorized.status, 403) + + const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged", { + headers: { "x-maple-maintenance-token": "maintenance-secret" }, + }) + const authorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + request, + new URL(request.url), + ) + strictEqual(authorized.status, 200) + deepStrictEqual(await authorized.json(), [{ id: "staged" }]) + }) + + it("evaluates and stages before chDB write, then marks ready before acknowledging", async () => { + const order: string[] = [] + const event = { id: "event-1" } + const db = { + exec: () => { + order.push("chdb-insert") + }, + } + const authority = { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => { + order.push("retention-filter") + return { ndjson, accepted: 1, rejected: 0 } + }, + } + const eventing = { + evaluateOtlp: () => { + order.push("evaluate") + return { events: [event], failures: [], typeMismatchFields: [] } + }, + persistFailures: () => order.push("persist-failures"), + stage: () => { + order.push("stage") + return { inserted: 1, deduplicated: 0, eventIds: [event.id] } + }, + markReady: () => order.push("ready"), + } + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786131720123456789", + body: { stringValue: "one" }, + }, + ], + }, + ], + }, + ], + }), + }) + + const result = await __testables.ingest( + db as never, + authority as never, + eventing as never, + "logs", + request, + ) + strictEqual(result.response.status, 200) + strictEqual(result.accepted, 1) + deepStrictEqual(order, [ + "evaluate", + "persist-failures", + "stage", + "retention-filter", + "chdb-insert", + "ready", + ]) + }) + + it("leaves a staged event non-ready when the warehouse write fails", async () => { + let markedReady = false + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "one" } }] }] }], + }), + }) + const result = await __testables.ingest( + { + exec: () => { + throw new Error("write failed") + }, + } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + { + evaluateOtlp: () => ({ events: [{ id: "event-1" }], failures: [], typeMismatchFields: [] }), + persistFailures: () => undefined, + stage: () => ({ inserted: 1, deduplicated: 0, eventIds: ["event-1"] }), + markReady: () => { + markedReady = true + }, + } as never, + "logs", + request, + ) + strictEqual(result.response.status, 500) + strictEqual(markedReady, false) + }) +}) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts new file mode 100644 index 000000000..30c1a7915 --- /dev/null +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -0,0 +1,203 @@ +import { deepStrictEqual, strictEqual } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import type { SignalProjectionSpec } from "@maple/eventing-core" +import { LocalEventingControlStore } from "../src/server/eventing/control-store" +import { normalizeOtlpLogs } from "../src/server/eventing/otlp" +import { LocalEventingRuntime } from "../src/server/eventing/runtime" +import { encodeLogs } from "../src/server/otlp/encode" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-runtime-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const attr = (key: string, value: Record) => ({ key, value }) + +const gitlabIssueCreated = { + resourceLogs: [ + { + resource: { + attributes: [ + attr("service.name", { stringValue: "gitlab-rails" }), + attr("service.version", { stringValue: "19.1.0" }), + ], + }, + scopeLogs: [ + { + scope: { name: "gitlab.event_store", version: "1.0.0" }, + logRecords: [ + { + timeUnixNano: "1786131720123456789", + observedTimeUnixNano: "1786131721123456789", + eventName: "gitlab.issue.created", + severityNumber: 9, + severityText: "INFO", + body: { stringValue: "Issue 42 created" }, + attributes: [ + attr("event.id", { stringValue: "01K20GITLABISSUE42" }), + attr("event.source", { stringValue: "https://gitlab.internal" }), + attr("gitlab.project.id", { intValue: "7" }), + attr("gitlab.project.path", { stringValue: "platform/maple" }), + attr("gitlab.issue.id", { intValue: "4200" }), + attr("gitlab.issue.iid", { intValue: "42" }), + attr("gitlab.issue.title", { stringValue: "Wire GitLab events" }), + attr("gitlab.issue.url", { + stringValue: "https://gitlab.internal/platform/maple/-/issues/42", + }), + attr("gitlab.user.id", { intValue: "9" }), + attr("gitlab.user.username", { stringValue: "operator" }), + ], + }, + ], + }, + ], + }, + ], +} + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: { + op: "all", + clauses: [ + { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + { + op: "gte", + field: { namespace: "attribute", key: "gitlab.issue.iid", type: "int64" }, + value: { type: "int64", value: "1" }, + }, + ], + }, + projector: { id: "gitlab.issue.created", version: 1, config: {} }, + activeFrom: "2000-01-01T00:00:00Z", + ...overrides, +}) + +describe("LocalEventingRuntime", () => { + it("normalizes typed GitLab OTLP fields while preserving the existing warehouse encoding", () => { + const [signal] = normalizeOtlpLogs(gitlabIssueCreated, "2026-08-07T20:00:00Z") + strictEqual(signal?.occurrenceId, "01K20GITLABISSUE42") + strictEqual(signal?.identityQuality, "source") + strictEqual(signal?.source, "https://gitlab.internal") + deepStrictEqual(signal?.fields.get("attribute:gitlab.issue.iid"), { + type: "int64", + value: "42", + }) + const batches = encodeLogs(gitlabIssueCreated) + strictEqual(batches.length, 1) + strictEqual(batches[0]?.rowCount, 1) + strictEqual(JSON.parse(batches[0]!.ndjson).log_attributes["gitlab.issue.iid"], "42") + }) + + it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + strictEqual(runtime.hasActiveSource("otel.log"), false) + runtime.activate(projection()) + const first = runtime.evaluateOtlp("logs", gitlabIssueCreated) + strictEqual(first.failures.length, 0) + strictEqual(first.events.length, 1) + deepStrictEqual(first.events[0], { + specversion: "1.0", + id: first.events[0]!.id, + source: "https://gitlab.internal", + type: "dev.maple.gitlab.issue.created.v1", + subject: "platform/maple/issues/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:gitlab-issue-created:v1", + tenantid: "local", + projectionid: "gitlab-issue-created", + projectionrevision: 1, + projectorid: "gitlab.issue.created", + projectorversion: 1, + data: { + project: { id: "7", path: "platform/maple" }, + issue: { + id: "4200", + iid: "42", + title: "Wire GitLab events", + url: "https://gitlab.internal/platform/maple/-/issues/42", + }, + actor: { id: "9", username: "operator" }, + serviceName: "gitlab-rails", + }, + }) + const staged = runtime.stage(first.events) + strictEqual(staged.inserted, 1) + strictEqual(runtime.listReady().length, 0) + deepStrictEqual(runtime.listStaged(), first.events) + const retry = runtime.evaluateOtlp("logs", gitlabIssueCreated) + strictEqual(retry.events[0]?.id, first.events[0]?.id) + strictEqual(runtime.stage(retry.events).deduplicated, 1) + runtime.markReady(staged.eventIds) + deepStrictEqual(runtime.listReady(), first.events) + deepStrictEqual(runtime.listStaged(), []) + } finally { + store.close() + } + })) + + it("activates a validated revision without restart and reloads it after restart", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + let runtime = new LocalEventingRuntime(store) + runtime.activate(projection()) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 1) + runtime.activate( + projection({ + revision: 2, + selector: { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.closed" }, + }, + }), + ) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 0) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + try { + runtime = new LocalEventingRuntime(store) + strictEqual(runtime.listActive()[0]?.revision, 2) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 0) + } finally { + store.close() + } + })) + + it("does no normalization or event work for a source with no active projection", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + deepStrictEqual(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") }), { + events: [], + failures: [], + typeMismatchFields: [], + }) + } finally { + store.close() + } + })) +}) diff --git a/apps/cli/test/server-network.test.ts b/apps/cli/test/server-network.test.ts index 53c413abb..46f389d8a 100644 --- a/apps/cli/test/server-network.test.ts +++ b/apps/cli/test/server-network.test.ts @@ -171,7 +171,8 @@ describe("browser origin policy", () => { deepStrictEqual(corsHeadersForAllowedOrigin(hostedOrigin), { "access-control-allow-origin": hostedOrigin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": "content-type, content-encoding", + "access-control-allow-headers": + "content-type, content-encoding, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", }) diff --git a/bun.lock b/bun.lock index 963a1178d..5d0242d77 100644 --- a/bun.lock +++ b/bun.lock @@ -59,6 +59,7 @@ "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", @@ -93,6 +94,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1", @@ -511,6 +513,9 @@ "packages/alerting-core": { "name": "@maple/alerting-core", "version": "0.0.0", + "dependencies": { + "@maple/eventing-core": "workspace:*", + }, "devDependencies": { "@types/node": "catalog:tooling", "typescript": "catalog:tooling", @@ -647,6 +652,19 @@ "react": "^19.0.0", }, }, + "packages/eventing-core": { + "name": "@maple/eventing-core", + "version": "0.0.0", + "dependencies": { + "effect": "catalog:effect", + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/infra": { "name": "@maple/infra", "devDependencies": { @@ -1555,6 +1573,8 @@ "@maple/email": ["@maple/email@workspace:packages/email"], + "@maple/eventing-core": ["@maple/eventing-core@workspace:packages/eventing-core"], + "@maple/infra": ["@maple/infra@workspace:packages/infra"], "@maple/ingest": ["@maple/ingest@workspace:apps/ingest"], diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md new file mode 100644 index 000000000..b1e1f91ce --- /dev/null +++ b/docs/signal-to-event-projection.md @@ -0,0 +1,881 @@ +# Signal-to-event projection architecture + +Status: implemented on `codex/issue-222-alerting-core`; downstream delivery remains out of scope + +Related work: [issue #222](https://github.com/MapleTechLabs/maple/issues/222), +`@maple/alerting-core` + +Audience: Maple maintainers and implementers of hosted or Maple Local runtimes + +## Decision summary + +Maple will treat immediate, per-occurrence event generation as an ingest concern, +not as a scheduled warehouse-query concern. + +- Each accepted OTLP record or provider webhook is decoded and normalized into a + typed signal once. +- An immutable snapshot of enabled signal projections is evaluated against that + signal before its scalar types are flattened for warehouse storage. +- Every matching projection invokes a registered, pure projector that produces a + factual [CloudEvents 1.0](https://github.com/cloudevents/spec/blob/main/cloudevents/spec.md) + event. +- Produced events enter a durable, idempotent outbox. Consumers and delivery + transports are downstream of that boundary. +- The original telemetry continues through the existing warehouse write path. +- chDB is not polled to discover newly arrived records. It remains the analytics + store and an optional, explicitly invoked replay source. +- Scheduled aggregate alerts remain query-driven. Alert lifecycle transitions + become another producer of typed events and use the same outbox as ingest-time + projections. + +The configurable matching model is a small, structured, typed predicate tree. It +is not arbitrary SQL and it is not a new textual expression language. The live +runtime evaluates the tree in memory. A warehouse adapter may lower the supported +subset to parameterized ClickHouse expressions for explicit historical replay, +but SQL behavior does not define the predicate semantics. + +## Problem + +Maple currently contains several mechanisms that are related but not expressed +through one event boundary: + +- hosted alert rules periodically query telemetry, update incident lifecycle + state, and request deliveries; +- PlanetScale receives signed webhooks and performs provider-specific work; +- Maple Local accepts OTLP records and writes them directly to chDB; +- future automation needs individual facts, such as a GitLab issue-created + signal, to become events that agents or other consumers can act on. + +Using the alert scheduler for the last case would give it the wrong semantics. +A windowed query answers a question about a set of stored records and normally +produces one aggregate observation. It cannot faithfully represent every +individual occurrence without cursors, overlap windows, late-arrival handling, +and deduplication. + +The current Local `logs` table has no ingestion sequence or native event ID. Its +sort key is designed for observability queries, and arbitrary OTLP attributes are +stored as strings. Repeatedly querying that table once per rule would therefore: + +- compete with ingest, UI queries, checkpoints, retention, and archive work; +- miss late records or repeatedly rediscover records unless a second deduplication + system is added; +- require casts that cannot always recover the source value's original type; +- turn an embedded analytical database into an inefficient message queue. + +The event layer is still useful. It belongs in front of chDB for live signals, +with chDB retained behind it for analytics and aggregate alert evaluation. + +## Goals + +1. Allow operators and integrations to configure which incoming signals become + typed events without writing SQL or changing core runtime code. +2. Evaluate each delivered signal in one ingest pass against all applicable + projections; do not issue one warehouse query per projection. +3. Preserve scalar types for string, boolean, integer, floating-point, + timestamp, and duration comparisons. +4. Make source adapters, projectors, event persistence, and consumers replaceable + behind explicit interfaces. +5. Give emitted events stable identities so retries do not create duplicate + logical events when the source provides stable occurrence identity. +6. Reuse the same event envelope and outbox for query-alert lifecycle events. +7. Keep Maple Local headless: matching and event persistence must work while no + browser is open. +8. Keep the core deterministic, bounded, tenant-scoped, and independent of a + database, network, scheduler, wall clock, or particular deployment host. + +## Non-goals + +- Adding NATS, JetStream, Kafka, or another general-purpose broker as a required + Maple component. +- Loading arbitrary third-party code into a running Maple process. A "plugin" in + this document is a compile-time registered module behind a stable interface. +- Defining sink delivery, Matrix behavior, agent authorization, or action policy. +- Replacing the Collector's routing, filtering, queueing, or authentication. +- Replacing scheduled queries for rates, percentiles, absence, threshold state, + or other aggregate alerts. +- Guaranteeing exactly-once external side effects across an uncooperative source, + Maple, and an arbitrary consumer. +- Automatically replaying old telemetry whenever a projection is created or + changed. +- Providing a general scripting language, joins, aggregation, arithmetic, + regular expressions, or user-provided SQL in the first version. + +## Terminology + +**Signal** +: One factual input occurrence after authentication, decoding, and normalization. +It may originate as an OTLP log/span/metric point or a provider webhook. + +**Source adapter** +: A module that verifies or accepts a source payload, normalizes occurrences into +typed signals, declares known fields, and supplies source identity when +available. + +**Signal projection** +: Durable configuration pairing a source kind, typed selector, and registered +projector. It says which source occurrences should be promoted into which +event representation. It is distinct from a downstream event subscription. + +**Selector** +: A bounded structured predicate over typed signal fields. + +**Projector** +: A pure, versioned function that maps one matching signal to a declared event +type and data schema. Provider-specific meaning belongs here rather than in the +eventing core. + +**Event** +: An immutable CloudEvents 1.0 envelope containing a typed factual payload. + +**Event outbox** +: Durable host storage that makes event creation idempotent and separates event +production from downstream delivery. + +**Event consumer** +: A downstream component interested in one or more event types. Webhooks, Matrix, +agents, issue creation, and existing provider responses are consumer concerns, +not selector or projector concerns. + +## Architecture + +There are two intentionally different event-production paths. They converge only +after a factual event has been produced. + +```mermaid +flowchart LR + Source["OTLP or provider source"] --> Gate["Authenticate / verify"] + Gate --> Decode["Decode once"] + Decode --> Signal["Typed normalized signal"] + + Signal --> Match["Ingest-time selector evaluation"] + Match --> Project["Registered signal projector"] + Project --> Outbox["Durable event outbox"] + + Signal --> Encode["Warehouse encoder"] + Encode --> Warehouse["chDB / hosted warehouse"] + + Warehouse --> Scheduled["Scheduled aggregate query"] + Scheduled --> Lifecycle["Alert evaluation and lifecycle"] + Lifecycle --> AlertProjector["Alert lifecycle projector"] + AlertProjector --> Outbox +``` + +The upper path handles occurrences such as "this GitLab issue was created". The +lower path handles conclusions such as "the error rate has remained above five +percent for ten minutes". Both can ultimately notify the same consumers without +pretending they have the same input or timing semantics. + +### Required module boundaries + +The architecture has four replaceable boundaries: + +1. **Source adapters** turn authenticated source payloads into typed signals. +2. **Selectors** determine whether a normalized signal qualifies. +3. **Projectors** map a qualifying signal to a typed factual event. +4. **Consumers** subscribe to event types downstream of the durable outbox. + +The eventing core owns the contracts and deterministic behavior. It does not know +about PlanetScale, GitLab, Matrix, chDB, PostgreSQL, Cloudflare Queues, or HTTP. + +PlanetScale is therefore one installed composition, not the model itself. Its +module can register a webhook source adapter and PlanetScale-specific projectors. +Those projectors can be replaced or supplemented without changing the selector +evaluator or downstream event contract. Existing PlanetScale behavior can later +be moved behind consumers of those typed events without putting provider actions +inside the projector. + +## Core data contracts + +The TypeScript below is illustrative. Canonical persisted encodings must be +defined with runtime schemas and shared conformance fixtures. + +### Typed values + +```ts +type SignalScalar = + | { readonly type: "string"; readonly value: string } + | { readonly type: "boolean"; readonly value: boolean } + | { readonly type: "int64"; readonly value: string } + | { readonly type: "float64"; readonly value: number } + | { readonly type: "timestamp"; readonly value: string } + | { readonly type: "duration"; readonly value: string } +``` + +`int64` and `duration` use decimal strings in serialized form so JavaScript does +not lose precision. Runtime evaluators may compile them to native `bigint` or the +equivalent host type. Timestamp values use canonical RFC 3339 with an explicit +offset in serialized form and compare as UTC instants. Duration values represent +integer nanoseconds. `float64` values must be finite; `NaN` and infinities are +rejected during normalization. + +Arrays and objects may be preserved for projector payloads, but selectors operate +only on declared scalar fields in version 1. + +### Normalized signal + +```ts +interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: unknown +} +``` + +- `sourceKind` chooses the compatible field catalog and projector registry. +- `source` is a stable URI identifying the logical producer or integration. +- `occurrenceId` is a source-issued stable identifier when one exists. +- `identityQuality: "source"` means the adapter expects the ID to survive source + retries and rebatching. `"derived"` identifies a canonical content fingerprint + with documented collision/collapse limitations. `"none"` cannot support a + durable once-only automation guarantee. +- `occurredAt` is source time; `observedAt` is Maple acceptance time. +- `fields` contains canonical built-ins and namespaced source attributes. It must + not contain secrets merely because they were present in the incoming payload. +- `data` is a bounded, schema-validated, source-specific representation available + to compatible projectors. It may contain arrays and objects that are not + selector-addressable, but it follows the adapter's redaction policy and is not + an unvalidated raw request body. + +The source adapter must not expose an unbounded raw payload as the selector field +space or projector input. + +### Field references and catalogs + +A selector uses logical field references, never physical column names: + +```ts +interface FieldRef { + readonly namespace: "signal" | "resource" | "scope" | "attribute" | "body" + readonly key: string + readonly type: SignalScalar["type"] +} +``` + +Each source adapter exposes a field catalog for known fields. A catalog entry +declares: + +- logical name and scalar type; +- allowed selector operators; +- sensitivity and whether a projector may expose it by default; +- whether historical replay is `exact`, `coerced`, or `unavailable`; +- an optional backend-owned replay binding. This binding is not user SQL. + +OTLP resource, scope, and record attributes are open-ended. A projection may +reference an uncatalogued attribute by explicitly declaring its expected scalar +type. At runtime a differently typed value does not get coerced; it does not +match, and a bounded type-mismatch metric is recorded. Source-specific modules +should publish catalogs for common attributes so users do not need to repeat +those declarations. + +### Selector AST + +```ts +type SignalPredicate = + | { readonly op: "all"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "any"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "not"; readonly clause: SignalPredicate } + | { readonly op: "exists"; readonly field: FieldRef } + | { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalScalar + } + | { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalScalar[] + } +``` + +Version 1 has the following semantics: + +| Operation | Supported types | Semantics | +| ------------------------ | ------------------------------------------- | ------------------------------------------------------------------------- | +| `exists` | all | True only when the field is present with a valid typed scalar. | +| `eq`, `neq` | all | Exact same-type comparison. A missing or mistyped field makes both false. | +| `gt`, `gte`, `lt`, `lte` | `int64`, `float64`, `timestamp`, `duration` | Ordered same-type comparison. | +| `contains` | `string` | Case-sensitive Unicode substring comparison. | +| `in` | all | Exact same-type membership; all literals must share the field type. | +| `all`, `any`, `not` | predicates | Total boolean composition with short-circuit evaluation. | + +There are no implicit casts. The string `"3"` is not the integer `3`; an integer +is not silently promoted to a float; and a string that resembles a date is not a +timestamp. Adapters may deliberately normalize a provider value into a declared +type, but that conversion is part of the source contract and is tested there. + +Missing values are not equivalent to null. Null source values are treated as +missing in version 1. Consequently `neq` requires a present field, whereas +`not(eq(...))` also matches a missing field. Configuration tooling should prefer +the explicit form that expresses the intended behavior. + +Validation happens before a projection can become active. Version 1 limits a +selector to: + +- nesting depth of 8; +- 64 total predicate nodes; +- 100 members in one `in` predicate; +- 4 KiB per string literal; +- no regular expressions, functions, arithmetic, joins, or user code. + +These bounds keep evaluation predictable and leave room for indexing active +projections by source kind and simple discriminating fields. + +### Signal projection + +```ts +interface SignalProjectionSpec { + readonly id: string + readonly revision: number + readonly enabled: boolean + readonly tenantId: string + readonly sourceKind: string + readonly selector: SignalPredicate + readonly projector: { + readonly id: string + readonly version: number + readonly config: unknown + } + readonly activeFrom: string +} +``` + +Every semantic edit creates a new immutable revision. Activation is not +retroactive: the new revision sees signals accepted after the runtime atomically +installs its compiled registry snapshot. Historical processing requires an +explicit replay operation. + +The configuration record is data. Source adapters and projector implementations +are registered code. This is how matching remains configurable without making +authentication, provider semantics, or executable code user-supplied. + +For example, the first GitLab projection uses the following concrete contract: + +```json +{ + "id": "gitlab-issue-created", + "revision": 1, + "enabled": true, + "tenantId": "local", + "sourceKind": "otel.log", + "selector": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "signal", "key": "event.name", "type": "string" }, + "value": { "type": "string", "value": "gitlab.issue.created" } + }, + { + "op": "gte", + "field": { "namespace": "attribute", "key": "gitlab.issue.iid", "type": "int64" }, + "value": { "type": "int64", "value": "1" } + } + ] + }, + "projector": { "id": "gitlab.issue.created", "version": 1, "config": {} }, + "activeFrom": "2026-08-07T00:00:00Z" +} +``` + +The `gte` comparison above is an integer comparison, not lexicographic string +ordering. A timestamp predicate would similarly carry a `timestamp` literal and +compare normalized instants rather than formatted text. No query is generated +for either comparison on the live path. + +### Projector contract + +```ts +interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly validateConfig: (value: unknown) => ProjectorConfig + readonly project: (signal: NormalizedSignal, config: ProjectorConfig) => ProjectedEventData +} +``` + +A projector must be pure, deterministic, bounded, versioned, and free of I/O. It +does not create issues, call Matrix, send notifications, or mutate provider +state. It produces a factual event payload conforming to its declared schema. + +The registry may include a bounded generic field-mapping projector for +operator-defined factual events. Provider modules register semantic projectors +when field copying is insufficient. No runtime module loading is required. + +### Event envelope + +Produced events use CloudEvents 1.0 structured representation: + +```json +{ + "specversion": "1.0", + "id": "sha256:...", + "source": "urn:maple:source:otel:local", + "type": "dev.maple.gitlab.issue.created.v1", + "subject": "project/example/issues/42", + "time": "2026-08-07T19:42:00.000000000Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-issue:v1", + "tenantid": "...", + "projectionid": "...", + "projectionrevision": 3, + "data": {} +} +``` + +Names above are illustrative until the repository reserves its canonical event +type and schema namespace. + +The event ID is deterministic when stable occurrence identity exists: + +```text +SHA-256(tenant ID, source kind, source URI, occurrence ID, projection ID, projection revision) +``` + +The hash input uses a canonical length-delimited encoding, not string +concatenation. Projector version and output schema version are already fixed by +the immutable projection revision and must be recorded with the event. + +Sensitive source details belong in `data`, under the projector's explicit schema +and redaction policy. They must not be copied into CloudEvents context attributes, +logs, metrics labels, or idempotency keys. + +## Runtime behavior + +### Projection compilation and activation + +The host loads enabled projections for a tenant, validates them against the +source and projector registries, and compiles them into immutable predicate +functions. The active registry is swapped atomically. Every decoded ingest batch +uses exactly one registry snapshot, even if configuration changes while the batch +is being processed. + +The initial implementation may evaluate all projections in the applicable +`sourceKind` bucket. The registry may later index projections by exact-match +discriminators such as event name or service name. This is an optimization and +must not alter selector semantics or ordering. + +Projection evaluation is deterministic and side-effect free. All matching +projections run; this is not first-match routing. A signal may therefore produce +zero, one, or several different factual events. + +### Maple Local OTLP ingest + +Maple Local already decodes an OTLP request and then passes the decoded payload +to the warehouse encoder. The event seam belongs between those operations. + +The implementation should refactor decoding/normalization so that: + +1. the OTLP request is parsed once; +2. typed record values remain available to the matcher; +3. the existing warehouse rows are produced without changing their stored shape; +4. matched events are staged idempotently before ingest acknowledges success; +5. the telemetry insert completes; +6. staged events are marked ready for downstream consumption; +7. only then is the OTLP request acknowledged. + +When no projection matches, the path adds only bounded predicate work before the +existing chDB insert. + +If the event store cannot stage a required event, ingest returns a retryable +failure rather than silently losing automation. A source retry reuses the same +event ID when stable occurrence identity is available, so staging is idempotent. + +Staging and chDB insertion are not one transaction. A process crash after the +chDB insert but before the OTLP acknowledgement can still cause a duplicate raw +telemetry row on retry; that is already possible with at-least-once OTLP +delivery. The staged/ready outbox protocol prevents an event from becoming +dispatchable before the ingest attempt reaches its warehouse commit point. + +If atomic exactly-once storage across both systems later becomes a requirement, +the correct addition is a durable ingress journal before both writes. chDB +polling does not solve that problem. + +### Provider webhooks + +Provider authentication and replay protection run before normalization. The +host must establish a durable event boundary before acknowledging the provider. +The hosted PlanetScale route therefore projects a verified payload first and +enqueues the resulting CloudEvent together with its temporary parity payload; +the queue is its durable event boundary. + +The provider source adapter supplies the strongest available delivery or event +identity. It then uses the same selector, projector, event ID, and outbox +contracts as OTLP. Provider-specific response behavior does not live in the core; +it can be migrated behind consumers of the emitted event types. + +### Query-driven alerts + +Scheduled alert rules retain their existing execution model: + +1. the host schedules and claims a rule; +2. a warehouse query produces an aggregate `AlertObservation`; +3. `@maple/alerting-core` evaluates threshold and lifecycle state; +4. an alert lifecycle projector converts `trigger`, `resolve`, `renotify`, or + `test` intent into a CloudEvent; +5. the host persists it through the common event outbox. + +This path queries chDB or the hosted warehouse because its input is an aggregate +over time. It does not reuse the ingest-time signal selector, and the ingest-time +path does not impersonate an alert incident. + +### Historical replay + +Replay is an operator-invoked batch operation, never the live event mechanism. +It evaluates one projection revision over a bounded time range and must support a +dry-run count/sample mode before it can persist events. + +Every field catalog entry declares replay capability: + +- `exact`: stored data retains enough type and identity information to reproduce + live semantics; +- `coerced`: the adapter can apply an explicit cast, but the source type was lost + or identity is derived; +- `unavailable`: the backend cannot implement the live predicate faithfully. + +A replay request using a `coerced` field requires explicit operator +acknowledgement. A request using an unavailable field is rejected. The warehouse +compiler emits parameterized expressions through existing query-building +facilities; it never interpolates field names or literals supplied directly by a +user. + +Current Local OTLP attribute maps store strings, so arbitrary typed attributes +will generally be `coerced`, not `exact`. Replay event IDs are guaranteed to +deduplicate against live events only when the warehouse retained the same stable +source occurrence ID. + +## Processing and delivery guarantees + +The architecture uses precise, layered guarantees rather than the blanket phrase +"exactly once". + +| Boundary | Guarantee | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Source to Maple | At least once when the source/Collector retries; source-specific otherwise. | +| One accepted batch | One evaluation against one immutable projection-registry snapshot. | +| Projection with source-stable identity | Effectively-once event creation through deterministic ID plus unique outbox insertion. | +| Projection with derived identity | Best-effort deduplication; identical real occurrences may collapse and re-encoded retries may diverge. | +| Projection with no identity | At-least-once event creation only; durable automation should reject this configuration by default. | +| Outbox to consumer | At least once with an event ID/idempotency key; consumer-side external effects are outside this specification. | +| chDB telemetry row | Existing OTLP semantics; duplicate storage remains possible after ambiguous failures. | + +A projection intended to trigger external automation must require +`identityQuality: "source"` unless an operator explicitly accepts weaker +semantics. GitLab event instrumentation should therefore furnish a stable event +or delivery identifier as part of its source contract. + +## chDB responsibilities + +chDB is responsible for: + +- storing telemetry for interactive and analytical queries; +- serving scheduled aggregate-alert queries; +- serving bounded explicit replay where field capabilities allow it; +- participating in existing checkpoint, retention, and archive workflows. + +chDB is not responsible for: + +- acting as a live queue; +- maintaining one cursor per signal projection; +- deduplicating event delivery; +- storing mutable projection configuration or delivery attempts merely because + it stores the source telemetry; +- defining selector type semantics through ClickHouse casts. + +Version 1 requires no new column or sort-key change to the existing telemetry +tables. A future narrow event journal or ingress-identity column may improve +replay, but it must be justified separately and must not turn wide raw-telemetry +tables into queue state. + +## Alternatives considered + +| Alternative | Decision | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| One periodic chDB query per projection | Rejected. It repeats wide scans, introduces cursor/late-arrival problems, and competes with the analytical workload. | +| One shared query that tails all recent chDB rows | Rejected as the live path. It reduces query count but still lacks a reliable ingestion cursor and evaluates after scalar type loss. It may inform an explicit replay implementation. | +| ClickHouse materialized views per projection | Rejected. Mutable user configuration would become DDL, current attribute storage has already flattened types, and lifecycle/deduplication state still needs another store. | +| Collector OTTL as Maple's rule language | Kept as an optional deployment optimization. It is valuable for OTel-only routing but does not define provider-webhook behavior or Maple-managed dynamic configuration. | +| CEL as the first expression language | Deferred. CEL is safe and capable, but embedding compatible runtimes and defining warehouse lowering is more surface than the initial predicates require. Reconsider it if the bounded AST is demonstrably insufficient. | +| CloudEvents SQL as the signal selector | Rejected for raw signals. [CESQL 1.0](https://github.com/cloudevents/spec/blob/main/cesql/spec.md) filters CloudEvent context attributes but does not address arbitrary event `data`; it may be useful for downstream CloudEvent subscriptions. | +| NATS or another broker as the event abstraction | Rejected as a requirement. A broker can later implement an event transport port, but it does not replace source normalization, selector semantics, projectors, identity, or host persistence. | +| A custom textual DSL | Rejected. The structured predicate tree is the persisted intermediate representation; configuration UIs and APIs do not need a parser. | + +## Durable host ports + +The core needs interfaces rather than a prescribed database: + +```ts +interface SignalProjectionStore { + loadEnabled(tenantId: string): Promise +} + +interface EventOutboxStore { + stage(events: readonly CloudEvent[]): Promise + markReady(eventIds: readonly string[]): Promise +} +``` + +The real contracts also need revision/change notification, unique event IDs, +bounded batch operations, health inspection, and recovery of staged records. + +Hosted Maple may implement these ports with its relational state and queue +infrastructure. Maple Local needs a small transactional control-state store whose +rules, outbox, and migration identity survive restart. That state is not covered +by chDB checkpoints automatically; backup, restore, and schema migration are part +of the Local host adapter's acceptance criteria. + +The physical Local store is an implementation decision, but it must provide: + +- uniqueness on event ID; +- atomic projection revision writes; +- atomic event staging and readiness transitions; +- bounded recovery of stranded staged events; +- crash-safe migrations and explicit backup/restore behavior; +- no dependency on a browser process. + +## Package and host ownership + +The intended ownership is: + +- `packages/eventing-core` (new): language-neutral schemas, selector validation, + the reference TypeScript evaluator, projector registry contracts, canonical + event identity, and conformance fixtures. No database, network, scheduler, or + global clock dependencies. +- `packages/alerting-core` (existing): aggregate alert evaluation and incident + lifecycle. It remains distinct and later emits through an eventing-core port. +- `packages/domain`: public/API schemas when projection CRUD becomes public. +- `apps/cli`: Maple Local OTLP source adapter, compiled-registry lifecycle, + durable Local ports, ingest staging, and optional replay adapter. +- `apps/api`: provider webhook adapters and hosted persistence wiring. +- `apps/ingest`: a future Rust OTLP adapter only when hosted per-signal projection + is required. + +The canonical JSON schemas and fixture corpus, rather than TypeScript source +types, define cross-language behavior. A Rust implementation must pass the same +valid/invalid selector cases, typed comparison cases, canonical event-ID vectors, +and projection fixtures before it can claim compatibility. + +[OpenTelemetry Transformation Language](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl) +can remain a Collector-side optimization or adapter. It is not the universal +Maple contract because it is coupled to OTel Collector contexts and does not +cover provider webhooks. [CEL](https://cel.dev/overview/cel-overview) is the +preferred language to reconsider if real requirements outgrow the bounded AST; +version 1 does not embed CEL runtimes or define a CEL-to-ClickHouse compiler. + +## Security and tenancy + +- Authentication or provider signature verification occurs before a source + adapter may produce a signal. +- Every signal, projection, event, and outbox operation carries an explicit + tenant ID. Cross-tenant registry lookup or event fanout is forbidden. +- User configuration cannot name SQL columns, inject SQL fragments, load code, + call functions, or select secrets outside the source field catalog. +- Source adapters mark sensitive fields. Generic projectors exclude them by + default; provider projectors must opt in deliberately and document why. +- Projected event size and source-field size are bounded before outbox insertion. +- Runtime errors and telemetry must not record full sensitive payloads. +- Sink URL validation, private-network policy, signing, and agent authorization + remain downstream policies. General eventing must not weaken hosted SSRF + protections. + +## Failure handling and observability + +Malformed projection configuration is rejected before activation. The reference +evaluator is total: missing fields and runtime type mismatches produce defined +non-matches rather than exceptions. + +A projector must return either schema-valid event data or a bounded typed +projection failure. A bad occurrence must not create an infinite source retry +loop. The host records the failure against projection ID/revision and occurrence +identity, exposes degraded health, and quarantines or dead-letters according to a +bounded policy. Exact quarantine policy belongs to the host adapter, but silently +dismissing a durable projection failure is not allowed. + +Required low-cardinality telemetry includes: + +- received signals by source kind; +- selector evaluations and matches by projection ID; +- selector type mismatches by source kind and field catalog key; +- projection failures; +- outbox staged, deduplicated, ready, and stranded counts; +- evaluation and staging latency; +- active projection count and registry revision; +- replay scanned, matched, emitted, and deduplicated counts. + +Raw field values, subjects, event IDs, and arbitrary event types must not become +unbounded metric labels. + +## Compatibility and migration + +This design extends rather than replaces the host-neutral alert-core extraction +already on the issue-222 branch. + +1. Existing hosted aggregate alerts continue using their scheduler, query, + lifecycle, and delivery behavior while the event contract is introduced. +2. The new eventing core lands without runtime activation and with conformance + fixtures. +3. Maple Local adds ingest-time projection behind an explicit feature/config + gate. With no active projections, observable ingest and chDB behavior remain + unchanged. +4. A GitLab issue-created OTLP fixture proves the end-to-end source identity, + typed selector, projector, retry deduplication, and durable outbox path. +5. PlanetScale is adapted behind the same source/projector interfaces while its + existing externally visible behavior remains intact. A compare/dual-observe + period should precede removal of direct hard-coded handling. +6. Alert lifecycle intents are projected into the same CloudEvents/outbox model + after parity tests show no change to trigger, resolve, renotify, test, + suppression, or retry semantics. +7. Warehouse replay is added only after the live path is proven and replay + capability metadata is implemented. + +No migration step requires NATS, a per-rule chDB cursor, or a new raw-telemetry +sort key. + +## Implementation slices for the next goal + +### Slice 1 — Contract and evaluator + +- Add `packages/eventing-core`. +- Define runtime schemas for typed values, fields, predicates, projection specs, + projector registrations, and CloudEvent output. +- Implement validation, compilation, and the pure reference evaluator. +- Add canonical JSON and event-ID test vectors. +- Add complexity-limit and hostile-input tests. + +### Slice 2 — Local durable control state + +- Select and document the Local transactional store. +- Implement projection revision and outbox ports, migrations, recovery, and + backup/restore hooks. +- Expose headless health inspection before UI work. + +### Slice 3 — Local ingest seam + +- Refactor OTLP normalization to preserve typed values without decoding twice. +- Load and atomically swap compiled projection snapshots. +- Stage matching events, insert telemetry, mark events ready, and acknowledge. +- Prove that the live path executes no chDB `SELECT` and adds no scheduler. + +### Slice 4 — First vertical: GitLab event to durable Maple event + +- Capture the real GitLab OTLP field contract and stable occurrence identity. +- Register its field catalog and issue-event projector. +- Configure an issue-created projection without hard-coded selector values. +- Verify duplicate source deliveries create one logical outbox event. + +This slice stops at the outbox. Matrix and agent-action behavior are a downstream +goal using the produced typed event. + +### Slice 5 — Existing producer convergence + +- Adapt PlanetScale webhook inputs to the source/projector contracts. +- Project alert lifecycle intents into CloudEvents. +- Preserve existing provider and alert behavior with parity fixtures before + switching consumers. + +### Slice 6 — Optional replay + +- Add per-field replay capability declarations. +- Implement bounded dry-run and explicit emission modes. +- Add evaluator-versus-ClickHouse conformance tests for every `exact` binding. + +## Acceptance criteria + +The first usable implementation is complete when all of the following are true: + +1. A configured GitLab issue-created OTLP signal is matched before chDB encoding + and produces a schema-valid CloudEvent while the telemetry record is still + stored normally. +2. Re-delivery of a source-stable occurrence produces the same event ID and one + logical outbox record. +3. A nonmatching signal performs no warehouse read and creates no event. +4. Several active projections are evaluated from one registry snapshot, and all + matches run. +5. Integer, float, timestamp, duration, boolean, and string truth-table fixtures + pass with no implicit coercion. +6. Projection changes are validated, revisioned, persisted, and activated + atomically without restarting Maple Local. +7. Rules and ready/staged outbox records survive process restart and participate + in documented backup and recovery. +8. chDB query alerts retain their existing aggregate and lifecycle behavior. +9. No implementation requires a browser, a new broker, arbitrary runtime code, + raw SQL configuration, or a per-projection chDB poller. +10. The event envelope and selector fixture corpus are sufficient for a second + language implementation to demonstrate semantic parity. + +## Settled implementation choices + +The TypeScript reference implementation settles the remaining host choices as +follows: + +- Maple Local stores projection revisions, failures, and the staged/ready outbox + in SQLite at `/control/eventing.sqlite`, using WAL and `synchronous = +FULL`. A version-2 Maple checkpoint contains `control.sqlite` beside the chDB + backup and binds its byte count, SHA-256 digest, schema version, and row counts + in the checkpoint manifest. Version-1 checkpoints remain readable and restore + an empty control store. +- The first GitLab instrumentation contract is an OTLP log whose LogRecord + `eventName` is `gitlab.issue.created`. `event.id` is the preferred stable + source occurrence identifier; `cloudevents.id` and `gitlab.event.id` are + accepted aliases. The semantic projector requires `gitlab.project.path` and + integer `gitlab.issue.iid`; it also recognizes project/issue IDs, title, URL, + and actor attributes. GitLab itself does not synthesize this contract merely + because Maple is running: the emitting instrumentation or adapter must attach + those fields. +- Maple-owned event types use `dev.maple.*.v1`; schemas use + `urn:maple:event-schema:*:v1`. The current vertical emits + `dev.maple.gitlab.issue.created.v1` with + `urn:maple:event-schema:gitlab-issue-created:v1`. +- Attribute strings are limited to 16 KiB, source/event identities to 256 + characters (long stable inputs are represented by a SHA-256 URN), each + attribute namespace to 256 entries, + nested values to depth 8 and 1,024 nodes, normalized source data to 256 KiB, + and a canonical outbox CloudEvent to 256 KiB. Secret-like attribute names are + excluded from the projection field and data views. +- The Local TypeScript path is the reference live implementation. Hosted Rust + ingest remains a later adapter and must pass the shared schemas and fixture + corpus before claiming parity. +- Verified non-test PlanetScale webhooks run through a registered + `planetscale.webhook` source adapter, selector, and projector before the route + acknowledges them. The dedicated Cloudflare Queue durably carries + `dev.maple.planetscale.webhook.received.v1`; its temporary provider payload + keeps the existing issue and timeline consumers behaviorally unchanged while + they migrate to the event contract. +- Hosted query-alert delivery rows remain that producer's durable outbox. Their + payload now includes an additive deterministic + `dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` CloudEvent while + retaining every legacy top-level delivery field. +- Historical replay execution remains deliberately unimplemented in this + change. Field catalogs already declare `exact`, `coerced`, or `unavailable`, + but Local's current arbitrary attribute maps have lost source scalar type and + its warehouse rows do not furnish a native occurrence ID. A later bounded, + operator-invoked replay adapter must require explicit coercion acknowledgement + and pass live-evaluator conformance tests; the live path never falls back to a + chDB poller in the meantime. +- Projector failures with a source occurrence ID are idempotent per projection + revision. Local retains a bounded newest 10,000 failure rows per tenant and + exposes the count through the authenticated headless health endpoint. A + projector failure does not retry a valid telemetry occurrence forever; + infrastructure failure to persist required state remains retryable. + +Maple Local activates immutable revisions with authenticated +`POST /local/eventing/projections`. The same maintenance credential protects +`GET /local/eventing/projections`, `/local/eventing/health`, and +`/local/eventing/outbox`. The outbox endpoint returns ready events by default; +`?state=staged` exposes bounded inspection of records stranded before the chDB +commit point. Re-delivery is the safe recovery operation: it deduplicates the +same staged event ID and promotes it only after the warehouse write succeeds. +Maple never blindly promotes an old staged record because, after a crash, the +control store alone cannot prove whether the corresponding chDB write committed. +Activation compiles the entire candidate registry +before the SQLite commit and swaps the immutable runtime snapshot while ingest +is quiesced, so a request observes exactly one registry version. diff --git a/packages/alerting-core/README.md b/packages/alerting-core/README.md index a464cce3a..0571b49e0 100644 --- a/packages/alerting-core/README.md +++ b/packages/alerting-core/README.md @@ -13,16 +13,28 @@ Current hosted adapters live in `apps/api` and are scheduled by queries, Local durable state, an in-process scheduler, and its own outbound URL policy without importing either hosted application. +This package covers scheduled aggregate alerts. Immediate per-occurrence events +use the separate ingest-time architecture described in +[`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md). +Both paths may ultimately publish through the same typed event outbox, but raw +signal matching does not poll chDB or impersonate an alert lifecycle. + The boundary is: - query adapter -> `AlertObservation`; - evaluation policy + observation -> `AlertEvaluation`; - persistence snapshot + evaluation -> `AlertLifecyclePlan`; -- host persists the plan and sends its optional `eventType` through a delivery - adapter; +- host persists the plan, projects its optional `eventType` into the common + CloudEvents envelope, and sends that event through a delivery adapter; - delivery adapters share idempotency-key and bounded retry policy helpers; - host clock supplies `nowMs`; the core never reads global time. Rule CRUD, storage schemas, scheduler claims, destination configuration, and delivery transports remain host concerns. This keeps Local UI work optional: the alert runtime can evaluate and deliver while no browser is open. + +Hosted alert delivery rows are the existing durable outbox for this producer. +Their additive `event` payload contains the deterministic +`dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` envelope; current +destinations continue to receive the legacy top-level payload fields during the +migration. diff --git a/packages/alerting-core/package.json b/packages/alerting-core/package.json index 0ce27ea84..9c7034969 100644 --- a/packages/alerting-core/package.json +++ b/packages/alerting-core/package.json @@ -10,6 +10,9 @@ "test": "vitest run", "typecheck": "tsc --noEmit" }, + "dependencies": { + "@maple/eventing-core": "workspace:*" + }, "devDependencies": { "@types/node": "catalog:tooling", "typescript": "catalog:tooling", diff --git a/packages/alerting-core/src/index.test.ts b/packages/alerting-core/src/index.test.ts index 9ad7c26dc..f0786e57a 100644 --- a/packages/alerting-core/src/index.test.ts +++ b/packages/alerting-core/src/index.test.ts @@ -6,6 +6,7 @@ import { interleaveAlertRulesByTenant, makeAlertDeliveryKey, planAlertLifecycle, + projectAlertLifecycleEvent, type AlertEvaluation, } from "./index" @@ -177,6 +178,39 @@ describe("interleaveAlertRulesByTenant", () => { }) describe("delivery policy", () => { + it("projects lifecycle intents into deterministic common CloudEvents", () => { + const input = { + tenantId: "org-1", + ruleId: "rule-1", + ruleName: "High errors", + incidentId: "incident-1", + eventType: "trigger" as const, + incidentStatus: "open", + groupKey: "checkout", + signalType: "error_rate", + severity: "critical", + comparator: "gt" as const, + threshold: 5, + thresholdUpper: null, + windowMinutes: 5, + value: 7.2, + sampleCount: 12, + occurredAtMs: 1_786_131_720_123, + } + const event = projectAlertLifecycleEvent(input) + expect(event).toEqual(projectAlertLifecycleEvent(input)) + expect(event).toMatchObject({ + type: "dev.maple.alert.lifecycle.trigger.v1", + subject: "alert-incidents/incident-1", + tenantid: "org-1", + projectionid: "alert-lifecycle", + data: { eventType: "trigger", incidentId: "incident-1" }, + }) + expect(() => projectAlertLifecycleEvent({ ...input, occurredAtMs: Number.MAX_SAFE_INTEGER })).toThrow( + "outside the supported date range", + ) + }) + it("builds stable idempotency keys", () => { expect(makeAlertDeliveryKey("incident", "destination", "trigger", 42)).toBe( "incident:destination:trigger:42", diff --git a/packages/alerting-core/src/index.ts b/packages/alerting-core/src/index.ts index 4130d25b6..6d5b523d9 100644 --- a/packages/alerting-core/src/index.ts +++ b/packages/alerting-core/src/index.ts @@ -1,3 +1,5 @@ +import { makeCloudEvent, type MapleCloudEvent } from "@maple/eventing-core" + export type AlertComparator = "gt" | "gte" | "lt" | "lte" | "eq" | "neq" | "between" | "not_between" export type AlertEvaluationStatus = "breached" | "healthy" | "skipped" @@ -139,6 +141,87 @@ export type AlertIncidentTransition = "none" | "opened" | "continued" | "resolve export type AlertNotificationSuppression = "flapping" | "flap_resolution" | null export type AlertLifecycleHold = "missing_telemetry" | null +export interface AlertLifecycleEventInput { + readonly tenantId: string + readonly ruleId: string + readonly ruleName: string + readonly incidentId: string | null + readonly eventType: AlertEventType + readonly incidentStatus: string + readonly groupKey: string | null + readonly signalType: string + readonly severity: string + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly windowMinutes: number + readonly value: number | null + readonly sampleCount: number | null + readonly occurredAtMs: number +} + +/** Project a query-alert lifecycle intent into the common factual event envelope. */ +export const projectAlertLifecycleEvent = (input: AlertLifecycleEventInput): MapleCloudEvent => { + if (!Number.isSafeInteger(input.occurredAtMs) || input.occurredAtMs < 0) + throw new Error("alert lifecycle event time must be a non-negative epoch millisecond") + const occurredAtDate = new Date(input.occurredAtMs) + if (Number.isNaN(occurredAtDate.getTime())) + throw new Error("alert lifecycle event time is outside the supported date range") + const occurredAt = occurredAtDate.toISOString() + const occurrenceId = `${input.incidentId ?? input.ruleId}:${input.eventType}:${input.occurredAtMs}` + return makeCloudEvent({ + signal: { + sourceKind: "alert.lifecycle", + source: `urn:maple:alert-rule:${input.ruleId}`, + tenantId: input.tenantId, + occurrenceId, + identityQuality: "source", + occurredAt, + observedAt: occurredAt, + subject: + input.incidentId === null + ? `alert-rules/${input.ruleId}` + : `alert-incidents/${input.incidentId}`, + fields: new Map(), + data: {}, + }, + projection: { + id: "alert-lifecycle", + revision: 1, + enabled: true, + tenantId: input.tenantId, + sourceKind: "alert.lifecycle", + selector: { + op: "exists", + field: { namespace: "signal", key: "event_type", type: "string" }, + }, + projector: { id: "alert.lifecycle", version: 1, config: {} }, + activeFrom: occurredAt, + }, + projectorId: "alert.lifecycle", + projectorVersion: 1, + outputType: `dev.maple.alert.lifecycle.${input.eventType}.v1`, + dataSchema: "urn:maple:event-schema:alert-lifecycle:v1", + data: { + eventType: input.eventType, + incidentId: input.incidentId, + incidentStatus: input.incidentStatus, + rule: { + id: input.ruleId, + name: input.ruleName, + signalType: input.signalType, + severity: input.severity, + groupKey: input.groupKey, + comparator: input.comparator, + threshold: input.threshold, + thresholdUpper: input.thresholdUpper, + windowMinutes: input.windowMinutes, + }, + observed: { value: input.value, sampleCount: input.sampleCount }, + }, + }) +} + export interface AlertLifecycleInput { readonly policy: AlertLifecyclePolicy readonly evaluation: AlertEvaluation diff --git a/packages/eventing-core/README.md b/packages/eventing-core/README.md new file mode 100644 index 000000000..9e5a883c3 --- /dev/null +++ b/packages/eventing-core/README.md @@ -0,0 +1,22 @@ +# `@maple/eventing-core` + +Host-neutral signal-to-event contracts and deterministic runtime semantics. + +The package owns typed signal values, bounded selectors, pure projector +registration, canonical event identity, and an immutable compiled projection +registry. It has no database, network, scheduler, or wall-clock dependency. A +host authenticates and normalizes source input, supplies durable projection and +outbox adapters, and decides when compiled registries become active. + +See [`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md) +for the architecture and acceptance contract. + +The versioned interoperability artifacts are generated under `schemas/`, with +valid comparison and identity vectors in `fixtures/v1.json`. Run `bun test` to +verify generated-schema drift, hostile selector bounds, typed comparison +semantics, deterministic event IDs, and projector isolation. + +The first host adapter is Maple Local in `apps/cli/src/server/eventing`. It uses +an authenticated configuration endpoint, a SQLite projection/outbox store, and +the pre-chDB OTLP seam. The package itself deliberately contains none of those +host decisions. diff --git a/packages/eventing-core/fixtures/v1.json b/packages/eventing-core/fixtures/v1.json new file mode 100644 index 000000000..dfc1c66da --- /dev/null +++ b/packages/eventing-core/fixtures/v1.json @@ -0,0 +1,177 @@ +{ + "version": 1, + "eventIdVectors": [ + { + "name": "tenant-scoped projected occurrence", + "input": { + "tenantId": "tenant-a", + "sourceKind": "otel.log", + "source": "urn:maple:source:otel:local", + "occurrenceId": "event-123", + "projectionId": "gitlab-issue-created", + "projectionRevision": 3 + }, + "output": "sha256:061c0b5d99b92ef65ab8813c6d84988e4b1582e705e0077c952e62a0e84b6b08" + } + ], + "predicateVectors": [ + { + "name": "int64 remains exact above JavaScript safe integer range", + "predicate": { + "op": "gt", + "field": { "namespace": "attribute", "key": "counter", "type": "int64" }, + "value": { "type": "int64", "value": "9007199254740992" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "counter", + "value": { "type": "int64", "value": "9007199254740993" } + } + ], + "matches": true + }, + { + "name": "timestamps compare as UTC instants", + "predicate": { + "op": "eq", + "field": { "namespace": "signal", "key": "occurred_at", "type": "timestamp" }, + "value": { "type": "timestamp", "value": "2026-08-07T19:42:00.123456789Z" } + }, + "fields": [ + { + "namespace": "signal", + "key": "occurred_at", + "value": { "type": "timestamp", "value": "2026-08-07T15:42:00.123456789-04:00" } + } + ], + "matches": true + }, + { + "name": "numeric strings do not coerce", + "predicate": { + "op": "gte", + "field": { "namespace": "attribute", "key": "attempt", "type": "int64" }, + "value": { "type": "int64", "value": "3" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "attempt", + "value": { "type": "string", "value": "12" } + } + ], + "matches": false, + "typeMismatches": ["attribute:attempt"] + }, + { + "name": "neq does not match a missing field", + "predicate": { + "op": "neq", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "value": { "type": "string", "value": "closed" } + }, + "fields": [], + "matches": false + }, + { + "name": "boolean composition and string containment", + "predicate": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "attribute", "key": "active", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + { + "op": "contains", + "field": { "namespace": "body", "key": "text", "type": "string" }, + "value": { "type": "string", "value": "issue created" } + } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "active", + "value": { "type": "boolean", "value": true } + }, + { + "namespace": "body", + "key": "text", + "value": { "type": "string", "value": "gitlab issue created successfully" } + } + ], + "matches": true + }, + { + "name": "float64 ordering is numeric", + "predicate": { + "op": "lt", + "field": { "namespace": "attribute", "key": "ratio", "type": "float64" }, + "value": { "type": "float64", "value": 10.25 } + }, + "fields": [ + { + "namespace": "attribute", + "key": "ratio", + "value": { "type": "float64", "value": 9.5 } + } + ], + "matches": true + }, + { + "name": "durations compare as exact nanoseconds", + "predicate": { + "op": "gte", + "field": { "namespace": "signal", "key": "duration", "type": "duration" }, + "value": { "type": "duration", "value": "1000000000" } + }, + "fields": [ + { + "namespace": "signal", + "key": "duration", + "value": { "type": "duration", "value": "1000000001" } + } + ], + "matches": true + }, + { + "name": "boolean equality has no string coercion", + "predicate": { + "op": "eq", + "field": { "namespace": "attribute", "key": "enabled", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + "fields": [ + { + "namespace": "attribute", + "key": "enabled", + "value": { "type": "string", "value": "true" } + } + ], + "matches": false, + "typeMismatches": ["attribute:enabled"] + }, + { + "name": "string membership is exact and case-sensitive", + "predicate": { + "op": "in", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "values": [ + { "type": "string", "value": "opened" }, + { "type": "string", "value": "closed" } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "state", + "value": { "type": "string", "value": "Closed" } + } + ], + "matches": false + } + ] +} diff --git a/packages/eventing-core/package.json b/packages/eventing-core/package.json new file mode 100644 index 000000000..3dd387ac9 --- /dev/null +++ b/packages/eventing-core/package.json @@ -0,0 +1,24 @@ +{ + "name": "@maple/eventing-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "schemas": "bun run scripts/generate-schemas.ts", + "schemas:check": "bun run scripts/generate-schemas.ts --check", + "test": "bun run schemas:check && vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "effect": "catalog:effect" + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/eventing-core/schemas/cloud-event.v1.schema.json b/packages/eventing-core/schemas/cloud-event.v1.schema.json new file mode 100644 index 000000000..adae790a5 --- /dev/null +++ b/packages/eventing-core/schemas/cloud-event.v1.schema.json @@ -0,0 +1,162 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:cloud-event:v1", + "$ref": "#/$defs/MapleCloudEvent", + "$defs": { + "MapleCloudEvent": { + "type": "object", + "properties": { + "specversion": { + "type": "string", + "enum": ["1.0"] + }, + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "subject": { + "type": "string" + }, + "time": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + }, + "datacontenttype": { + "type": "string", + "enum": ["application/json"] + }, + "dataschema": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "tenantid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionrevision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "projectorid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectorversion": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "data": {} + }, + "required": [ + "specversion", + "id", + "source", + "type", + "time", + "datacontenttype", + "dataschema", + "tenantid", + "projectionid", + "projectionrevision", + "projectorid", + "projectorversion", + "data" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-projection.v1.schema.json b/packages/eventing-core/schemas/signal-projection.v1.schema.json new file mode 100644 index 000000000..44f2adbae --- /dev/null +++ b/packages/eventing-core/schemas/signal-projection.v1.schema.json @@ -0,0 +1,349 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-projection:v1", + "$ref": "#/$defs/SignalProjectionSpec", + "$defs": { + "SignalFieldRef": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "enum": ["signal", "resource", "scope", "attribute", "body"] + }, + "key": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 512 + } + ] + }, + "type": { + "type": "string", + "enum": ["string", "boolean", "int64", "float64", "timestamp", "duration"] + } + }, + "required": ["namespace", "key", "type"], + "additionalProperties": false + }, + "SignalScalar": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + }, + "SignalPredicate": { + "anyOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["all"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + } + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["any"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + } + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["not"] + }, + "clause": { + "$ref": "#/$defs/SignalPredicate" + } + }, + "required": ["op", "clause"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["exists"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + } + }, + "required": ["op", "field"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["eq", "neq", "gt", "gte", "lt", "lte", "contains"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "value": { + "$ref": "#/$defs/SignalScalar" + } + }, + "required": ["op", "field", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["in"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalScalar" + } + } + }, + "required": ["op", "field", "values"], + "additionalProperties": false + } + ] + }, + "SignalProjectionSpec": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "revision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "enabled": { + "type": "boolean" + }, + "tenantId": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "sourceKind": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "selector": { + "$ref": "#/$defs/SignalPredicate" + }, + "projector": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "version": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "config": {} + }, + "required": ["id", "version", "config"], + "additionalProperties": false + }, + "activeFrom": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": [ + "id", + "revision", + "enabled", + "tenantId", + "sourceKind", + "selector", + "projector", + "activeFrom" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-scalar.v1.schema.json b/packages/eventing-core/schemas/signal-scalar.v1.schema.json new file mode 100644 index 000000000..bbf5af1c1 --- /dev/null +++ b/packages/eventing-core/schemas/signal-scalar.v1.schema.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-scalar:v1", + "$ref": "#/$defs/SignalScalar", + "$defs": { + "SignalScalar": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + } + } +} diff --git a/packages/eventing-core/scripts/generate-schemas.ts b/packages/eventing-core/scripts/generate-schemas.ts new file mode 100644 index 000000000..2ea54a516 --- /dev/null +++ b/packages/eventing-core/scripts/generate-schemas.ts @@ -0,0 +1,61 @@ +import { execFileSync } from "node:child_process" +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { dirname, resolve } from "node:path" +import { Schema } from "effect" +import { MapleCloudEventSchema, SignalProjectionSpecSchema, SignalScalarSchema } from "../src/model" + +const root = resolve(import.meta.dirname, "..") +const check = process.argv.includes("--check") + +const documents = [ + { + path: "schemas/signal-scalar.v1.schema.json", + id: "urn:maple:eventing:schema:signal-scalar:v1", + schema: SignalScalarSchema, + }, + { + path: "schemas/signal-projection.v1.schema.json", + id: "urn:maple:eventing:schema:signal-projection:v1", + schema: SignalProjectionSpecSchema, + }, + { + path: "schemas/cloud-event.v1.schema.json", + id: "urn:maple:eventing:schema:cloud-event:v1", + schema: MapleCloudEventSchema, + }, +] as const + +let stale = false +for (const entry of documents) { + const document = Schema.toJsonSchemaDocument(entry.schema) + const unformatted = `${JSON.stringify( + { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: entry.id, + ...document.schema, + ...(Object.keys(document.definitions).length === 0 ? {} : { $defs: document.definitions }), + }, + null, + "\t", + )}\n` + const serialized = execFileSync( + resolve(root, "../../node_modules/.bin/oxfmt"), + ["--stdin-filepath", entry.path], + { + input: unformatted, + encoding: "utf8", + }, + ) + const path = resolve(root, entry.path) + if (check) { + if (!existsSync(path) || readFileSync(path, "utf8") !== serialized) { + console.error(`${entry.path} is stale; run bun run schemas`) + stale = true + } + } else { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, serialized) + } +} + +if (stale) process.exitCode = 1 diff --git a/packages/eventing-core/src/event.ts b/packages/eventing-core/src/event.ts new file mode 100644 index 000000000..e70308e78 --- /dev/null +++ b/packages/eventing-core/src/event.ts @@ -0,0 +1,117 @@ +import { createHash } from "node:crypto" +import type { JsonValue, MapleCloudEvent, NormalizedSignal, SignalProjectionSpec } from "./model" +import { timestampToEpochNanos } from "./predicate" + +export interface EventIdentityInput { + readonly tenantId: string + readonly sourceKind: string + readonly source: string + readonly occurrenceId: string + readonly projectionId: string + readonly projectionRevision: number +} + +const updateLengthDelimited = (hash: ReturnType, value: string): void => { + const encoded = Buffer.from(value, "utf8") + const length = Buffer.allocUnsafe(4) + length.writeUInt32BE(encoded.byteLength) + hash.update(length) + hash.update(encoded) +} + +/** Canonical v1 identity shared by every host implementation. */ +export const makeEventId = (input: EventIdentityInput): string => { + const hash = createHash("sha256") + for (const field of [ + "maple-event-v1", + input.tenantId, + input.sourceKind, + input.source, + input.occurrenceId, + input.projectionId, + String(input.projectionRevision), + ]) + updateLengthDelimited(hash, field) + return `sha256:${hash.digest("hex")}` +} + +export const isJsonValue = (value: unknown, seen: Set = new Set()): value is JsonValue => { + if (value === null || typeof value === "string" || typeof value === "boolean") return true + if (typeof value === "number") return Number.isFinite(value) + if (typeof value !== "object") return false + if (seen.has(value)) return false + seen.add(value) + try { + if (Array.isArray(value)) return value.every((item) => isJsonValue(item, seen)) + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return false + return Object.values(value).every((item) => isJsonValue(item, seen)) + } finally { + // Track the active recursion path. Repeated references serialize as a + // JSON tree and are not themselves cycles. + seen.delete(value) + } +} + +const canonicalizeJson = (value: JsonValue): JsonValue => { + if (value === null || typeof value !== "object") return value + if (Array.isArray(value)) return value.map(canonicalizeJson) + const record = value as { readonly [key: string]: JsonValue } + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalizeJson(record[key]!)]), + ) +} + +/** Stable JSON encoding for outbox collision checks and cross-host fixtures. */ +export const canonicalJson = (value: JsonValue): string => { + if (!isJsonValue(value)) throw new Error("value must be finite acyclic JSON") + return JSON.stringify(canonicalizeJson(value)) +} + +export const makeCloudEvent = (input: { + readonly signal: NormalizedSignal + readonly projection: SignalProjectionSpec + readonly projectorId: string + readonly projectorVersion: number + readonly outputType: string + readonly dataSchema: string + readonly subject?: string | null + readonly time?: string + readonly data: JsonValue +}): MapleCloudEvent => { + if (input.signal.occurrenceId === null || input.signal.identityQuality === "none") + throw new Error("durable event projection requires stable or derived occurrence identity") + if (!isJsonValue(input.data)) throw new Error("projected event data must be finite JSON") + if (input.outputType.length === 0) throw new Error("projected event type must not be empty") + if (input.dataSchema.length === 0) throw new Error("projected event data schema must not be empty") + if (input.signal.source.length === 0) throw new Error("signal source must not be empty") + + const subject = input.subject ?? input.signal.subject + const time = input.time ?? input.signal.occurredAt + if (timestampToEpochNanos(time) === null) throw new Error("projected event time must be a valid instant") + return { + specversion: "1.0", + id: makeEventId({ + tenantId: input.signal.tenantId, + sourceKind: input.signal.sourceKind, + source: input.signal.source, + occurrenceId: input.signal.occurrenceId, + projectionId: input.projection.id, + projectionRevision: input.projection.revision, + }), + source: input.signal.source, + type: input.outputType, + ...(subject == null ? {} : { subject }), + time, + datacontenttype: "application/json", + dataschema: input.dataSchema, + tenantid: input.signal.tenantId, + projectionid: input.projection.id, + projectionrevision: input.projection.revision, + projectorid: input.projectorId, + projectorversion: input.projectorVersion, + data: input.data, + } +} diff --git a/packages/eventing-core/src/index.ts b/packages/eventing-core/src/index.ts new file mode 100644 index 000000000..87253218d --- /dev/null +++ b/packages/eventing-core/src/index.ts @@ -0,0 +1,5 @@ +export * from "./event" +export * from "./model" +export * from "./predicate" +export * from "./registry" +export * from "./source" diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts new file mode 100644 index 000000000..9b291b5e4 --- /dev/null +++ b/packages/eventing-core/src/model.ts @@ -0,0 +1,219 @@ +import { Schema } from "effect" + +const NonEmptyIdentifier = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(256), + Schema.isTrimmed(), +) + +const DecimalInt64 = Schema.String.check(Schema.isPattern(/^-?(?:0|[1-9][0-9]*)$/)) + +const Rfc3339Timestamp = Schema.String.check( + Schema.isPattern(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/), +) + +export const StringSignalScalar = Schema.Struct({ + type: Schema.Literal("string"), + value: Schema.String, +}) + +export const BooleanSignalScalar = Schema.Struct({ + type: Schema.Literal("boolean"), + value: Schema.Boolean, +}) + +export const Int64SignalScalar = Schema.Struct({ + type: Schema.Literal("int64"), + value: DecimalInt64, +}) + +export const Float64SignalScalar = Schema.Struct({ + type: Schema.Literal("float64"), + value: Schema.Finite, +}) + +export const TimestampSignalScalar = Schema.Struct({ + type: Schema.Literal("timestamp"), + value: Rfc3339Timestamp, +}) + +export const DurationSignalScalar = Schema.Struct({ + type: Schema.Literal("duration"), + value: DecimalInt64, +}) + +export const SignalScalarSchema = Schema.Union([ + StringSignalScalar, + BooleanSignalScalar, + Int64SignalScalar, + Float64SignalScalar, + TimestampSignalScalar, + DurationSignalScalar, +]).annotate({ identifier: "SignalScalar" }) +export type SignalScalar = Schema.Schema.Type +export type SignalScalarType = SignalScalar["type"] + +export const FieldNamespaceSchema = Schema.Literals(["signal", "resource", "scope", "attribute", "body"]) +export type FieldNamespace = Schema.Schema.Type + +export const FieldRefSchema = Schema.Struct({ + namespace: FieldNamespaceSchema, + key: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(512)), + type: Schema.Literals(["string", "boolean", "int64", "float64", "timestamp", "duration"]), +}).annotate({ identifier: "SignalFieldRef" }) +export type FieldRef = Schema.Schema.Type + +export interface AllPredicate { + readonly op: "all" + readonly clauses: readonly SignalPredicate[] +} + +export interface AnyPredicate { + readonly op: "any" + readonly clauses: readonly SignalPredicate[] +} + +export interface NotPredicate { + readonly op: "not" + readonly clause: SignalPredicate +} + +export interface ExistsPredicate { + readonly op: "exists" + readonly field: FieldRef +} + +export interface ComparisonPredicate { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalScalar +} + +export interface InPredicate { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalScalar[] +} + +export type SignalPredicate = + | AllPredicate + | AnyPredicate + | NotPredicate + | ExistsPredicate + | ComparisonPredicate + | InPredicate + +export const SignalPredicateSchema: Schema.Codec = Schema.suspend( + (): Schema.Codec => + Schema.Union([ + Schema.Struct({ + op: Schema.Literal("all"), + clauses: Schema.Array(SignalPredicateSchema), + }), + Schema.Struct({ + op: Schema.Literal("any"), + clauses: Schema.Array(SignalPredicateSchema), + }), + Schema.Struct({ + op: Schema.Literal("not"), + clause: SignalPredicateSchema, + }), + Schema.Struct({ + op: Schema.Literal("exists"), + field: FieldRefSchema, + }), + Schema.Struct({ + op: Schema.Literals(["eq", "neq", "gt", "gte", "lt", "lte", "contains"]), + field: FieldRefSchema, + value: SignalScalarSchema, + }), + Schema.Struct({ + op: Schema.Literal("in"), + field: FieldRefSchema, + values: Schema.Array(SignalScalarSchema), + }), + ]) as Schema.Codec, +).annotate({ identifier: "SignalPredicate" }) + +export const ProjectorRefSchema = Schema.Struct({ + id: NonEmptyIdentifier, + version: Schema.Int.check(Schema.isGreaterThan(0)), + config: Schema.Unknown, +}) +export type ProjectorRef = Schema.Schema.Type + +export const SignalProjectionSpecSchema = Schema.Struct({ + id: NonEmptyIdentifier, + revision: Schema.Int.check(Schema.isGreaterThan(0)), + enabled: Schema.Boolean, + tenantId: NonEmptyIdentifier, + sourceKind: NonEmptyIdentifier, + selector: SignalPredicateSchema, + projector: ProjectorRefSchema, + activeFrom: Rfc3339Timestamp, +}).annotate({ identifier: "SignalProjectionSpec" }) +export type SignalProjectionSpec = Schema.Schema.Type + +export interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: TData +} + +export type JsonPrimitive = string | number | boolean | null +export type JsonValue = JsonPrimitive | { readonly [key: string]: JsonValue } | readonly JsonValue[] + +export interface ProjectedEventData { + readonly subject?: string | null + readonly time?: string + readonly data: JsonValue +} + +export interface MapleCloudEvent { + readonly specversion: "1.0" + readonly id: string + readonly source: string + readonly type: string + readonly subject?: string + readonly time: string + readonly datacontenttype: "application/json" + readonly dataschema: string + readonly tenantid: string + readonly projectionid: string + readonly projectionrevision: number + readonly projectorid: string + readonly projectorversion: number + readonly data: JsonValue +} + +export const MapleCloudEventSchema = Schema.Struct({ + specversion: Schema.Literal("1.0"), + id: NonEmptyIdentifier, + source: NonEmptyIdentifier, + type: NonEmptyIdentifier, + subject: Schema.optionalKey(Schema.String), + time: Rfc3339Timestamp, + datacontenttype: Schema.Literal("application/json"), + dataschema: NonEmptyIdentifier, + tenantid: NonEmptyIdentifier, + projectionid: NonEmptyIdentifier, + projectionrevision: Schema.Int.check(Schema.isGreaterThan(0)), + projectorid: NonEmptyIdentifier, + projectorversion: Schema.Int.check(Schema.isGreaterThan(0)), + data: Schema.Unknown, +}).annotate({ identifier: "MapleCloudEvent" }) + +export const fieldKey = (field: Pick): string => + `${field.namespace}:${field.key}` + +export const defineSignalFields = ( + fields: ReadonlyArray<{ readonly field: FieldRef; readonly value: SignalScalar }>, +): ReadonlyMap => + new Map(fields.map(({ field, value }) => [fieldKey(field), value] as const)) diff --git a/packages/eventing-core/src/predicate.test.ts b/packages/eventing-core/src/predicate.test.ts new file mode 100644 index 000000000..c4cf98362 --- /dev/null +++ b/packages/eventing-core/src/predicate.test.ts @@ -0,0 +1,148 @@ +import { readFileSync } from "node:fs" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" +import { + compileSignalPredicate, + defineSignalFields, + fieldKey, + makeEventId, + MAX_PREDICATE_DEPTH, + SignalPredicateSchema, + SignalScalarSchema, + timestampToEpochNanos, + validateSignalPredicate, + type EventIdentityInput, + type FieldNamespace, + type FieldRef, + type NormalizedSignal, +} from "./index" + +interface ConformanceFixture { + readonly eventIdVectors: ReadonlyArray<{ + readonly name: string + readonly input: EventIdentityInput + readonly output: string + }> + readonly predicateVectors: ReadonlyArray<{ + readonly name: string + readonly predicate: unknown + readonly fields: ReadonlyArray<{ + readonly namespace: FieldNamespace + readonly key: string + readonly value: unknown + }> + readonly matches: boolean + readonly typeMismatches?: readonly string[] + }> +} + +const fixture = JSON.parse( + readFileSync(new URL("../fixtures/v1.json", import.meta.url), "utf8"), +) as ConformanceFixture + +const signalFor = (fields: ConformanceFixture["predicateVectors"][number]["fields"]): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "occurrence-1", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00Z", + observedAt: "2026-08-07T19:42:01Z", + subject: null, + fields: defineSignalFields( + fields.map(({ namespace, key, value }) => ({ + field: { + namespace, + key, + type: Schema.decodeUnknownSync(SignalScalarSchema)(value).type, + }, + value: Schema.decodeUnknownSync(SignalScalarSchema)(value), + })), + ), + data: {}, +}) + +describe("cross-language conformance vectors", () => { + for (const vector of fixture.eventIdVectors) { + it(`event ID: ${vector.name}`, () => { + expect(makeEventId(vector.input)).toBe(vector.output) + }) + } + + for (const vector of fixture.predicateVectors) { + it(`predicate: ${vector.name}`, () => { + const predicate = Schema.decodeUnknownSync(SignalPredicateSchema)(vector.predicate) + const result = compileSignalPredicate(predicate)(signalFor(vector.fields)) + expect(result.matches).toBe(vector.matches) + expect(result.typeMismatches.map(fieldKey)).toEqual(vector.typeMismatches ?? []) + }) + } +}) + +describe("selector validation", () => { + it("rejects wrong literal types and unsupported ordering", () => { + const field: FieldRef = { namespace: "attribute", key: "enabled", type: "boolean" } + expect( + validateSignalPredicate({ op: "gt", field, value: { type: "string", value: "true" } }), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ message: "gt is not supported for boolean" }), + expect.objectContaining({ message: "field and literal types must match" }), + ]), + ) + }) + + it("rejects empty combinators and excessive nesting", () => { + expect(validateSignalPredicate({ op: "all", clauses: [] })).toContainEqual({ + path: "selector.clauses", + message: "all requires at least one clause", + }) + + let nested = { + op: "exists" as const, + field: { namespace: "attribute" as const, key: "x", type: "string" as const }, + } + for (let i = 0; i < MAX_PREDICATE_DEPTH; i++) nested = { op: "not", clause: nested } as never + expect(validateSignalPredicate(nested)).toEqual( + expect.arrayContaining([expect.objectContaining({ message: `predicate depth exceeds 8` })]), + ) + }) + + it("rejects invalid calendar dates and int64 overflow", () => { + expect(timestampToEpochNanos("2026-02-31T00:00:00Z")).toBeNull() + expect( + validateSignalPredicate({ + op: "eq", + field: { namespace: "attribute", key: "n", type: "int64" }, + value: { type: "int64", value: "9223372036854775808" }, + }), + ).toContainEqual( + expect.objectContaining({ message: "int64 must be a signed 64-bit decimal integer" }), + ) + }) +}) + +describe("total runtime behavior", () => { + it("treats malformed source scalars as mismatches rather than throwing", () => { + const field: FieldRef = { namespace: "attribute", key: "n", type: "int64" } + const evaluate = compileSignalPredicate({ + op: "gte", + field, + value: { type: "int64", value: "1" }, + }) + const signal = signalFor([]) + const fields = new Map(signal.fields) + fields.set(fieldKey(field), { type: "int64", value: "not-an-integer" }) + expect(evaluate({ ...signal, fields })).toMatchObject({ + matches: false, + typeMismatches: [field], + }) + }) + + it("distinguishes neq from not(eq) for a missing field", () => { + const field: FieldRef = { namespace: "attribute", key: "state", type: "string" } + const eq = { op: "eq" as const, field, value: { type: "string" as const, value: "closed" } } + expect(compileSignalPredicate({ ...eq, op: "neq" })(signalFor([])).matches).toBe(false) + expect(compileSignalPredicate({ op: "not", clause: eq })(signalFor([])).matches).toBe(true) + }) +}) diff --git a/packages/eventing-core/src/predicate.ts b/packages/eventing-core/src/predicate.ts new file mode 100644 index 000000000..ede5adf13 --- /dev/null +++ b/packages/eventing-core/src/predicate.ts @@ -0,0 +1,341 @@ +import type { + FieldRef, + NormalizedSignal, + SignalPredicate, + SignalProjectionSpec, + SignalScalar, + SignalScalarType, +} from "./model" +import { fieldKey } from "./model" + +export const MAX_PREDICATE_DEPTH = 8 +export const MAX_PREDICATE_NODES = 64 +export const MAX_IN_VALUES = 100 +export const MAX_STRING_LITERAL_BYTES = 4 * 1024 + +const INT64_MIN = -(1n << 63n) +const INT64_MAX = (1n << 63n) - 1n +const ORDERED_TYPES = new Set(["int64", "float64", "timestamp", "duration"]) + +export interface ValidationIssue { + readonly path: string + readonly message: string +} + +export class SignalPredicateValidationError extends Error { + readonly issues: readonly ValidationIssue[] + + constructor(issues: readonly ValidationIssue[]) { + super(issues.map(({ path, message }) => `${path}: ${message}`).join("; ")) + this.name = "SignalPredicateValidationError" + this.issues = issues + } +} + +const stringBytes = (value: string): number => new TextEncoder().encode(value).byteLength + +const parseInt64 = (value: string): bigint | null => { + try { + const parsed = BigInt(value) + return parsed >= INT64_MIN && parsed <= INT64_MAX ? parsed : null + } catch { + return null + } +} + +const isLeapYear = (year: number): boolean => year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) + +const daysInMonth = (year: number, month: number): number => { + switch (month) { + case 2: + return isLeapYear(year) ? 29 : 28 + case 4: + case 6: + case 9: + case 11: + return 30 + default: + return 31 + } +} + +const TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|([+-])(\d{2}):(\d{2}))$/ + +/** Parse the v1 RFC 3339 subset into exact UTC nanoseconds. */ +export const timestampToEpochNanos = (value: string): bigint | null => { + const match = TIMESTAMP.exec(value) + if (!match) return null + const year = Number(match[1]) + const month = Number(match[2]) + const day = Number(match[3]) + const hour = Number(match[4]) + const minute = Number(match[5]) + const second = Number(match[6]) + const fraction = match[7] ?? "" + if ( + month < 1 || + month > 12 || + day < 1 || + day > daysInMonth(year, month) || + hour > 23 || + minute > 59 || + second > 59 + ) + return null + + let offsetMinutes = 0 + if (match[8] !== "Z") { + const offsetHours = Number(match[10]) + const offsetMinutePart = Number(match[11]) + if (offsetHours > 23 || offsetMinutePart > 59) return null + offsetMinutes = offsetHours * 60 + offsetMinutePart + if (match[9] === "-") offsetMinutes = -offsetMinutes + } + + const date = new Date(0) + date.setUTCFullYear(year, month - 1, day) + date.setUTCHours(hour, minute, second, 0) + const milliseconds = date.getTime() - offsetMinutes * 60_000 + if (!Number.isFinite(milliseconds)) return null + const nanos = BigInt(fraction.padEnd(9, "0")) + return BigInt(milliseconds) * 1_000_000n + nanos +} + +export const validateSignalScalar = (scalar: SignalScalar, path = "value"): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + switch (scalar.type) { + case "string": + if (stringBytes(scalar.value) > MAX_STRING_LITERAL_BYTES) + issues.push({ path, message: `string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes` }) + break + case "boolean": + break + case "int64": + case "duration": + if (parseInt64(scalar.value) === null) + issues.push({ path, message: `${scalar.type} must be a signed 64-bit decimal integer` }) + break + case "float64": + if (!Number.isFinite(scalar.value)) issues.push({ path, message: "float64 must be finite" }) + break + case "timestamp": + if (timestampToEpochNanos(scalar.value) === null) + issues.push({ + path, + message: "timestamp must be a valid RFC 3339 instant with an explicit offset", + }) + break + } + return issues +} + +export const validateSignalPredicate = (predicate: SignalPredicate): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + let nodes = 0 + + const visit = (node: SignalPredicate, path: string, depth: number): void => { + nodes += 1 + if (nodes > MAX_PREDICATE_NODES) return + if (depth > MAX_PREDICATE_DEPTH) { + issues.push({ path, message: `predicate depth exceeds ${MAX_PREDICATE_DEPTH}` }) + return + } + + switch (node.op) { + case "all": + case "any": + if (node.clauses.length === 0) + issues.push({ + path: `${path}.clauses`, + message: `${node.op} requires at least one clause`, + }) + for (let i = 0; i < node.clauses.length; i++) + visit(node.clauses[i]!, `${path}.clauses[${i}]`, depth + 1) + break + case "not": + visit(node.clause, `${path}.clause`, depth + 1) + break + case "exists": + break + case "contains": + if (node.field.type !== "string" || node.value.type !== "string") + issues.push({ path, message: "contains requires a string field and string literal" }) + issues.push(...validateSignalScalar(node.value, `${path}.value`)) + break + case "gt": + case "gte": + case "lt": + case "lte": + if (!ORDERED_TYPES.has(node.field.type)) + issues.push({ path, message: `${node.op} is not supported for ${node.field.type}` }) + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalScalar(node.value, `${path}.value`)) + break + case "eq": + case "neq": + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalScalar(node.value, `${path}.value`)) + break + case "in": + if (node.values.length === 0) + issues.push({ path: `${path}.values`, message: "in requires at least one value" }) + if (node.values.length > MAX_IN_VALUES) + issues.push({ path: `${path}.values`, message: `in exceeds ${MAX_IN_VALUES} values` }) + for (let i = 0; i < node.values.length; i++) { + const value = node.values[i]! + if (value.type !== node.field.type) + issues.push({ + path: `${path}.values[${i}]`, + message: "field and literal types must match", + }) + issues.push(...validateSignalScalar(value, `${path}.values[${i}]`)) + } + break + } + } + + visit(predicate, "selector", 1) + if (nodes > MAX_PREDICATE_NODES) + issues.push({ path: "selector", message: `predicate exceeds ${MAX_PREDICATE_NODES} nodes` }) + return issues +} + +export const assertValidSignalPredicate = (predicate: SignalPredicate): void => { + const issues = validateSignalPredicate(predicate) + if (issues.length > 0) throw new SignalPredicateValidationError(issues) +} + +export const validateSignalProjectionSpec = ( + projection: SignalProjectionSpec, +): readonly ValidationIssue[] => [ + ...(timestampToEpochNanos(projection.activeFrom) === null + ? [{ path: "activeFrom", message: "must be a valid RFC 3339 instant with an explicit offset" }] + : []), + ...validateSignalPredicate(projection.selector), +] + +export interface PredicateEvaluation { + readonly matches: boolean + readonly typeMismatches: readonly FieldRef[] +} + +const scalarEquals = (left: SignalScalar, right: SignalScalar): boolean => { + if (left.type !== right.type) return false + switch (left.type) { + case "string": + return right.type === "string" && left.value === right.value + case "boolean": + return right.type === "boolean" && left.value === right.value + case "float64": + return right.type === "float64" && left.value === right.value + case "int64": + return right.type === "int64" && BigInt(left.value) === BigInt(right.value) + case "duration": + return right.type === "duration" && BigInt(left.value) === BigInt(right.value) + case "timestamp": + return ( + right.type === "timestamp" && + timestampToEpochNanos(left.value) === timestampToEpochNanos(right.value) + ) + } +} + +const scalarOrder = (left: SignalScalar, right: SignalScalar): number | null => { + if (left.type !== right.type || !ORDERED_TYPES.has(left.type)) return null + switch (left.type) { + case "int64": { + if (right.type !== "int64") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "duration": { + if (right.type !== "duration") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "float64": + return right.type !== "float64" + ? null + : left.value < right.value + ? -1 + : left.value > right.value + ? 1 + : 0 + case "timestamp": { + if (right.type !== "timestamp") return null + const a = timestampToEpochNanos(left.value)! + const b = timestampToEpochNanos(right.value)! + return a < b ? -1 : a > b ? 1 : 0 + } + default: + return null + } +} + +export type CompiledSignalPredicate = (signal: NormalizedSignal) => PredicateEvaluation + +export const compileSignalPredicate = (predicate: SignalPredicate): CompiledSignalPredicate => { + assertValidSignalPredicate(predicate) + + return (signal) => { + const typeMismatches: FieldRef[] = [] + const readField = (field: FieldRef): SignalScalar | undefined => { + const value = signal.fields.get(fieldKey(field)) + if (value === undefined) return undefined + if (value.type !== field.type || validateSignalScalar(value).length > 0) { + typeMismatches.push(field) + return undefined + } + return value + } + + const evaluate = (node: SignalPredicate): boolean => { + switch (node.op) { + case "all": + return node.clauses.every(evaluate) + case "any": + return node.clauses.some(evaluate) + case "not": + return !evaluate(node.clause) + case "exists": { + return readField(node.field) !== undefined + } + case "eq": + case "neq": + case "gt": + case "gte": + case "lt": + case "lte": + case "contains": { + const value = readField(node.field) + if (value === undefined) return false + if (node.op === "eq") return scalarEquals(value, node.value) + if (node.op === "neq") return !scalarEquals(value, node.value) + if (node.op === "contains") + return ( + value.type === "string" && + node.value.type === "string" && + value.value.includes(node.value.value) + ) + const order = scalarOrder(value, node.value) + if (order === null) return false + if (node.op === "gt") return order > 0 + if (node.op === "gte") return order >= 0 + if (node.op === "lt") return order < 0 + return order <= 0 + } + case "in": { + const value = readField(node.field) + if (value === undefined) return false + return node.values.some((candidate) => scalarEquals(value, candidate)) + } + } + } + + return { matches: evaluate(predicate), typeMismatches } + } +} diff --git a/packages/eventing-core/src/registry.test.ts b/packages/eventing-core/src/registry.test.ts new file mode 100644 index 000000000..e7bfd3a9a --- /dev/null +++ b/packages/eventing-core/src/registry.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it } from "vitest" +import { + CompiledProjectionRegistry, + canonicalJson, + defineSignalFields, + makeEventId, + ProjectorRegistry, + SignalSourceRegistry, + type NormalizedSignal, + type SignalProjectionSpec, +} from "./index" + +const signal = (overrides: Partial = {}): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "event-123", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00.123456789Z", + observedAt: "2026-08-07T19:42:01Z", + subject: "project/example/issues/42", + fields: defineSignalFields([ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + ]), + data: { issue: { iid: 42, title: "Example" } }, + ...overrides, +}) + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 3, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + projector: { id: "gitlab.issue", version: 1, config: { includeTitle: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const projectors = (): ProjectorRegistry => + new ProjectorRegistry().register({ + id: "gitlab.issue", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.created.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue:v1", + decodeConfig: (value) => { + if (typeof value !== "object" || value === null) throw new Error("invalid projector config") + return value + }, + project: (input) => ({ data: input.data as { issue: { iid: number; title: string } } }), + }) + +const sources = (): SignalSourceRegistry => + new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + openFields: [ + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64", "timestamp", "duration"], + operators: ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + +describe("CompiledProjectionRegistry", () => { + it("canonicalizes JSON independently of object insertion order", () => { + expect(canonicalJson({ z: 1, nested: { b: true, a: [2, 1] }, a: "first" })).toBe( + '{"a":"first","nested":{"a":[2,1],"b":true},"z":1}', + ) + const shared = { value: 1 } + expect(canonicalJson({ left: shared, right: shared })).toBe( + '{"left":{"value":1},"right":{"value":1}}', + ) + const cyclic: { self?: unknown } = {} + cyclic.self = cyclic + expect(() => canonicalJson(cyclic as never)).toThrow("finite acyclic JSON") + expect(() => canonicalJson({ invalid: Number.NaN })).toThrow("finite acyclic JSON") + }) + + it("projects every match into a deterministic CloudEvent", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const first = registry.evaluate(signal()) + const second = registry.evaluate(signal()) + expect(first.failures).toEqual([]) + expect(first.events).toEqual(second.events) + expect(first.events).toHaveLength(1) + expect(first.events[0]).toMatchObject({ + specversion: "1.0", + id: makeEventId({ + tenantId: "tenant-a", + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + occurrenceId: "event-123", + projectionId: "gitlab-issue-created", + projectionRevision: 3, + }), + type: "dev.maple.gitlab.issue.created.v1", + subject: "project/example/issues/42", + projectionrevision: 3, + data: signal().data, + }) + }) + + it("runs every matching projection from one immutable registry snapshot", () => { + const registry = CompiledProjectionRegistry.compile( + [projection(), projection({ id: "gitlab-issue-created-audit" })], + sources(), + projectors(), + ) + const result = registry.evaluate(signal()) + expect(result.failures).toEqual([]) + expect(result.events.map(({ projectionid }) => projectionid)).toEqual([ + "gitlab-issue-created", + "gitlab-issue-created-audit", + ]) + }) + + it("runs all matching projections and isolates projector failures", () => { + const registryDefinitions = projectors().register({ + id: "broken", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.broken.v1", + dataSchema: "urn:maple:event-schema:broken:v1", + decodeConfig: () => ({}), + project: () => { + throw new Error("projector invariant failed") + }, + }) + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "broken-projection", projector: { id: "broken", version: 1, config: {} } }), + ], + sources(), + registryDefinitions, + ) + const result = registry.evaluate(signal()) + expect(result.events).toHaveLength(1) + expect(result.failures).toEqual([ + expect.objectContaining({ + projectionId: "broken-projection", + message: "projector invariant failed", + }), + ]) + }) + + it("isolates tenants, source kinds, activation time, and disabled revisions", () => { + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "future", revision: 1, activeFrom: "2026-08-08T00:00:00Z" }), + projection({ id: "disabled", revision: 1, enabled: false }), + projection({ id: "other-tenant", revision: 1, tenantId: "tenant-b" }), + ], + sources(), + projectors(), + ) + expect(registry.evaluate(signal()).events.map(({ projectionid }) => projectionid)).toEqual([ + "gitlab-issue-created", + ]) + expect(registry.evaluate(signal({ sourceKind: "otel.span" })).events).toEqual([]) + }) + + it("requires occurrence identity for durable projection", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const result = registry.evaluate(signal({ occurrenceId: null, identityQuality: "none" })) + expect(result.events).toEqual([]) + expect(result.failures[0]?.message).toBe( + "durable event projection requires stable or derived occurrence identity", + ) + }) + + it("rejects duplicate registrations, projection revisions, and invalid projector bindings", () => { + const definitions = projectors() + expect(() => + definitions.register({ + id: "gitlab.issue", + version: 1, + sourceKinds: ["otel.log"], + outputType: "duplicate", + dataSchema: "duplicate", + decodeConfig: (value) => value, + project: () => ({ data: {} }), + }), + ).toThrow("duplicate projector registration") + expect(() => + CompiledProjectionRegistry.compile([projection(), projection()], sources(), projectors()), + ).toThrow("duplicate projection revision") + expect(() => + CompiledProjectionRegistry.compile( + [projection({ projector: { id: "missing", version: 1, config: {} } })], + sources(), + projectors(), + ), + ).toThrow("unregistered projector") + }) + + it("validates selector fields and operators against the source catalog", () => { + const closed = new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["eq"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "contains", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("contains is not allowed for catalog field") + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "eq", + field: { namespace: "attribute", key: "unknown", type: "string" }, + value: { type: "string", value: "x" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("unknown field attribute:unknown") + }) +}) diff --git a/packages/eventing-core/src/registry.ts b/packages/eventing-core/src/registry.ts new file mode 100644 index 000000000..b5eb66adb --- /dev/null +++ b/packages/eventing-core/src/registry.ts @@ -0,0 +1,184 @@ +import { makeCloudEvent } from "./event" +import { Schema } from "effect" +import type { + JsonValue, + MapleCloudEvent, + NormalizedSignal, + ProjectedEventData, + SignalProjectionSpec, +} from "./model" +import { SignalProjectionSpecSchema } from "./model" +import { timestampToEpochNanos, compileSignalPredicate, validateSignalProjectionSpec } from "./predicate" +import { SignalSourceRegistry, validatePredicateAgainstSource } from "./source" + +export interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly decodeConfig: (value: unknown) => TConfig + readonly project: (signal: NormalizedSignal, config: TConfig) => ProjectedEventData +} + +type ErasedSignalProjector = SignalProjector + +export class ProjectorRegistry { + readonly #projectors = new Map() + + register(projector: SignalProjector): this { + if (projector.id.trim().length === 0) throw new Error("projector ID must not be empty") + if (!Number.isSafeInteger(projector.version) || projector.version < 1) + throw new Error("projector version must be a positive safe integer") + if (projector.sourceKinds.length === 0) throw new Error("projector must accept a source kind") + if (projector.outputType.length === 0) throw new Error("projector output type must not be empty") + if (projector.dataSchema.length === 0) throw new Error("projector data schema must not be empty") + const key = ProjectorRegistry.key(projector.id, projector.version) + if (this.#projectors.has(key)) throw new Error(`duplicate projector registration: ${key}`) + this.#projectors.set(key, projector as ErasedSignalProjector) + return this + } + + get(id: string, version: number): ErasedSignalProjector | undefined { + return this.#projectors.get(ProjectorRegistry.key(id, version)) + } + + static key(id: string, version: number): string { + return `${id}@${version}` + } +} + +interface CompiledProjection { + readonly spec: SignalProjectionSpec + readonly evaluate: ReturnType + readonly projector: ErasedSignalProjector + readonly config: unknown + readonly activeFromNanos: bigint +} + +export interface ProjectionFailure { + readonly projectionId: string + readonly projectionRevision: number + readonly occurrenceId: string | null + readonly message: string +} + +export interface ProjectionBatchResult { + readonly events: readonly MapleCloudEvent[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +const validateProjectedData = (value: ProjectedEventData): ProjectedEventData => { + if (value.time !== undefined && timestampToEpochNanos(value.time) === null) + throw new Error("projector returned an invalid event timestamp") + return value +} + +/** Immutable compiled snapshot. Hosts atomically replace the whole instance. */ +export class CompiledProjectionRegistry { + readonly #bySourceKind: ReadonlyMap + + private constructor(bySourceKind: ReadonlyMap) { + this.#bySourceKind = bySourceKind + } + + static compile( + specs: readonly SignalProjectionSpec[], + sources: SignalSourceRegistry, + projectors: ProjectorRegistry, + ): CompiledProjectionRegistry { + const bySourceKind = new Map() + const revisions = new Set() + + for (const candidate of specs) { + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + const source = sources.get(spec.sourceKind) + if (!source) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered source ${spec.sourceKind}`, + ) + const issues = [ + ...validateSignalProjectionSpec(spec), + ...validatePredicateAgainstSource(spec.selector, source), + ] + if (issues.length > 0) + throw new Error( + `invalid projection ${spec.id}@${spec.revision}: ${issues + .map(({ path, message }) => `${path}: ${message}`) + .join("; ")}`, + ) + const revisionKey = `${spec.tenantId}:${spec.id}@${spec.revision}` + if (revisions.has(revisionKey)) throw new Error(`duplicate projection revision: ${revisionKey}`) + revisions.add(revisionKey) + if (!spec.enabled) continue + + const projector = projectors.get(spec.projector.id, spec.projector.version) + if (!projector) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered projector ${spec.projector.id}@${spec.projector.version}`, + ) + if (!projector.sourceKinds.includes(spec.sourceKind)) + throw new Error( + `projector ${projector.id}@${projector.version} does not accept ${spec.sourceKind}`, + ) + + const compiled: CompiledProjection = { + spec, + evaluate: compileSignalPredicate(spec.selector), + projector, + config: projector.decodeConfig(spec.projector.config), + activeFromNanos: timestampToEpochNanos(spec.activeFrom)!, + } + const bucket = bySourceKind.get(spec.sourceKind) + if (bucket) bucket.push(compiled) + else bySourceKind.set(spec.sourceKind, [compiled]) + } + + return new CompiledProjectionRegistry(bySourceKind) + } + + evaluate(signal: NormalizedSignal): ProjectionBatchResult { + const events: MapleCloudEvent[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + const observedAtNanos = timestampToEpochNanos(signal.observedAt) + + for (const projection of this.#bySourceKind.get(signal.sourceKind) ?? []) { + if (projection.spec.tenantId !== signal.tenantId) continue + if (observedAtNanos === null || observedAtNanos < projection.activeFromNanos) continue + const evaluation = projection.evaluate(signal) + for (const field of evaluation.typeMismatches) + typeMismatchFields.add(`${field.namespace}:${field.key}`) + if (!evaluation.matches) continue + + try { + const projected = validateProjectedData( + projection.projector.project(signal, projection.config), + ) + events.push( + makeCloudEvent({ + signal, + projection: projection.spec, + projectorId: projection.projector.id, + projectorVersion: projection.projector.version, + outputType: projection.projector.outputType, + dataSchema: projection.projector.dataSchema, + subject: projected.subject, + time: projected.time, + data: projected.data as JsonValue, + }), + ) + } catch (error) { + failures.push({ + projectionId: projection.spec.id, + projectionRevision: projection.spec.revision, + occurrenceId: signal.occurrenceId, + message: error instanceof Error ? error.message : String(error), + }) + } + } + + return { events, failures, typeMismatchFields: [...typeMismatchFields] } + } +} diff --git a/packages/eventing-core/src/source.ts b/packages/eventing-core/src/source.ts new file mode 100644 index 000000000..a9a6a7ee2 --- /dev/null +++ b/packages/eventing-core/src/source.ts @@ -0,0 +1,141 @@ +import type { FieldNamespace, FieldRef, NormalizedSignal, SignalPredicate, SignalScalarType } from "./model" +import { fieldKey } from "./model" +import type { ValidationIssue } from "./predicate" + +export type SignalLeafOperator = "exists" | "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" | "in" +export type ReplayCapability = "exact" | "coerced" | "unavailable" + +export interface SignalFieldCatalogEntry { + readonly field: FieldRef + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export interface OpenFieldNamespacePolicy { + readonly namespace: FieldNamespace + readonly types: readonly SignalScalarType[] + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export interface SignalSourceDefinition { + readonly sourceKind: string + readonly fields: readonly SignalFieldCatalogEntry[] + readonly openFields?: readonly OpenFieldNamespacePolicy[] +} + +export interface SignalSourceAdapter { + readonly definition: SignalSourceDefinition + readonly normalize: (raw: TRaw, context: TContext) => readonly NormalizedSignal[] +} + +interface RegisteredSignalSource { + readonly definition: SignalSourceDefinition + readonly fields: ReadonlyMap + readonly openFields: ReadonlyMap +} + +export class SignalSourceRegistry { + readonly #sources = new Map() + + register(definition: SignalSourceDefinition): this { + if (definition.sourceKind.trim().length === 0) throw new Error("source kind must not be empty") + if (this.#sources.has(definition.sourceKind)) + throw new Error(`duplicate source registration: ${definition.sourceKind}`) + + const fields = new Map() + for (const entry of definition.fields) { + const key = fieldKey(entry.field) + if (fields.has(key)) + throw new Error(`duplicate field catalog entry: ${definition.sourceKind}:${key}`) + if (entry.operators.length === 0) throw new Error(`field catalog entry has no operators: ${key}`) + fields.set(key, entry) + } + + const openFields = new Map() + for (const policy of definition.openFields ?? []) { + if (openFields.has(policy.namespace)) + throw new Error(`duplicate open field policy: ${definition.sourceKind}:${policy.namespace}`) + if (policy.types.length === 0 || policy.operators.length === 0) + throw new Error(`open field policy must declare types and operators: ${policy.namespace}`) + openFields.set(policy.namespace, policy) + } + + this.#sources.set(definition.sourceKind, { definition, fields, openFields }) + return this + } + + get(sourceKind: string): RegisteredSignalSource | undefined { + return this.#sources.get(sourceKind) + } +} + +const leafFields = ( + predicate: SignalPredicate, +): ReadonlyArray<{ + readonly field: FieldRef + readonly operator: SignalLeafOperator + readonly path: string +}> => { + const fields: Array<{ field: FieldRef; operator: SignalLeafOperator; path: string }> = [] + const visit = (node: SignalPredicate, path: string): void => { + switch (node.op) { + case "all": + case "any": + for (let i = 0; i < node.clauses.length; i++) visit(node.clauses[i]!, `${path}.clauses[${i}]`) + break + case "not": + visit(node.clause, `${path}.clause`) + break + default: + fields.push({ field: node.field, operator: node.op, path }) + } + } + visit(predicate, "selector") + return fields +} + +export const validatePredicateAgainstSource = ( + predicate: SignalPredicate, + source: RegisteredSignalSource, +): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + for (const leaf of leafFields(predicate)) { + const catalogEntry = source.fields.get(fieldKey(leaf.field)) + if (catalogEntry) { + if (catalogEntry.field.type !== leaf.field.type) + issues.push({ + path: `${leaf.path}.field.type`, + message: `catalog field ${fieldKey(leaf.field)} has type ${catalogEntry.field.type}`, + }) + if (!catalogEntry.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for catalog field ${fieldKey(leaf.field)}`, + }) + continue + } + + const open = source.openFields.get(leaf.field.namespace) + if (!open) { + issues.push({ + path: `${leaf.path}.field`, + message: `unknown field ${fieldKey(leaf.field)} for source ${source.definition.sourceKind}`, + }) + continue + } + if (!open.types.includes(leaf.field.type)) + issues.push({ + path: `${leaf.path}.field.type`, + message: `${leaf.field.type} is not allowed for open ${leaf.field.namespace} fields`, + }) + if (!open.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for open ${leaf.field.namespace} fields`, + }) + } + return issues +} diff --git a/packages/eventing-core/tsconfig.json b/packages/eventing-core/tsconfig.json new file mode 100644 index 000000000..12d9920b4 --- /dev/null +++ b/packages/eventing-core/tsconfig.json @@ -0,0 +1,23 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022", "DOM"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true, + "plugins": [ + { + "name": "@effect/language-service", + "reportSuggestionsAsWarningsInTsc": true + } + ] + } +} From dae7c4452d871da3c8d1ae7fb6ddd20fb401cbf9 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 11 Aug 2026 16:51:52 -0400 Subject: [PATCH 03/12] fix(eventing): harden durability and compatibility --- .../src/planetscale-webhook-runtime.test.ts | 26 +++ apps/api/src/planetscale-webhook-runtime.ts | 16 +- .../src/services/alerts/AlertsService.test.ts | 35 ++++ apps/api/src/services/alerts/AlertsService.ts | 9 +- .../planetscale/PlanetScaleWebhookQueue.ts | 16 +- apps/cli/src/server/eventing/control-store.ts | 162 ++++++++++++++---- apps/cli/src/server/eventing/otlp.ts | 18 +- apps/cli/src/server/eventing/runtime.ts | 38 +++- apps/cli/src/server/serve.ts | 121 +++++++++++-- .../test/local-eventing-control-store.test.ts | 81 ++++++++- apps/cli/test/local-eventing-ingest.test.ts | 131 +++++++++++++- apps/cli/test/local-eventing-runtime.test.ts | 49 +++++- docs/signal-to-event-projection.md | 43 +++-- .../schemas/signal-projection.v1.schema.json | 43 ++++- .../schemas/signal-scalar.v1.schema.json | 13 +- packages/eventing-core/src/event.ts | 44 ++++- packages/eventing-core/src/model.ts | 28 ++- packages/eventing-core/src/predicate.test.ts | 35 ++++ packages/eventing-core/src/predicate.ts | 80 ++++++++- packages/eventing-core/src/registry.test.ts | 51 ++++++ packages/eventing-core/src/registry.ts | 6 +- 21 files changed, 921 insertions(+), 124 deletions(-) diff --git a/apps/api/src/planetscale-webhook-runtime.test.ts b/apps/api/src/planetscale-webhook-runtime.test.ts index 858b76567..08398f091 100644 --- a/apps/api/src/planetscale-webhook-runtime.test.ts +++ b/apps/api/src/planetscale-webhook-runtime.test.ts @@ -84,6 +84,32 @@ describe("PlanetScale webhook queue consumer", () => { }).pipe(Effect.provide(testDb.layer)) }) + it.effect("processes the exact pre-event-envelope queue body during rolling upgrades", () => { + const testDb = createTestDb(trackedDbs) + const legacyJob = { + kind: "planetscale-webhook", + orgId, + connectionId: "connection_1", + payload: job.payload, + receivedAt: 1_000, + } + const delivery = makeBatch(legacyJob) + return Effect.gen(function* () { + yield* processPlanetScaleWebhookBatch(delivery.batch) + assert.isTrue(delivery.acknowledged()) + assert.isFalse(delivery.retried()) + const row = yield* Effect.promise(() => + queryFirstRow<{ workflow_state: string; occurrence_count: number }>( + testDb, + "SELECT workflow_state, occurrence_count FROM error_issues WHERE org_id = $1", + ["org_1"], + ), + ) + assert.strictEqual(row?.workflow_state, "triage") + assert.strictEqual(row?.occurrence_count, 1) + }).pipe(Effect.provide(testDb.layer)) + }) + it.effect("acknowledges terminal malformed jobs", () => { const testDb = createTestDb(trackedDbs) const delivery = makeBatch({ kind: "not-a-planetscale-job" }) diff --git a/apps/api/src/planetscale-webhook-runtime.ts b/apps/api/src/planetscale-webhook-runtime.ts index 5a9e80fd2..544f702c5 100644 --- a/apps/api/src/planetscale-webhook-runtime.ts +++ b/apps/api/src/planetscale-webhook-runtime.ts @@ -10,9 +10,10 @@ import { deployRequestNumber, insertPlanetScaleEvent, planetScaleBranchName, + projectPlanetScaleWebhookEvent, upsertPlanetScaleIssue, } from "./services/integrations/planetscale/webhook-events" -import { PlanetScaleWebhookJob } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" +import { PlanetScaleWebhookQueueMessage } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" const telemetry = MapleCloudflareSDK.make({ serviceName: "maple-api", @@ -32,7 +33,7 @@ export const buildPlanetScaleWebhookLayer = (_env: Record) => { export const flushPlanetScaleWebhookTelemetry = (env: Record) => telemetry.flush(env) -const decodeJob = Schema.decodeUnknownEffect(PlanetScaleWebhookJob) +const decodeJob = Schema.decodeUnknownEffect(PlanetScaleWebhookQueueMessage) export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => Effect.forEach( @@ -54,7 +55,16 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => ), ), onSuccess: (job) => { - const event = job.event + // Old jobs can remain in Cloudflare Queue across a deploy. Rebuild the + // event from the durable legacy fields instead of malformed-acking them. + const event = + job.event ?? + projectPlanetScaleWebhookEvent({ + orgId: job.orgId, + connectionId: job.connectionId, + payload: job.payload, + receivedAt: job.receivedAt, + }) const eventData = typeof event.data === "object" && event.data !== null && diff --git a/apps/api/src/services/alerts/AlertsService.test.ts b/apps/api/src/services/alerts/AlertsService.test.ts index b7c434fd6..f3e98ca61 100644 --- a/apps/api/src/services/alerts/AlertsService.test.ts +++ b/apps/api/src/services/alerts/AlertsService.test.ts @@ -1,6 +1,7 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { Cause, Clock, ConfigProvider, Duration, Effect, Exit, Layer, Option, Schema } from "effect" import { TestClock } from "effect/testing" +import { projectAlertLifecycleEvent } from "@maple/alerting-core" import { AlertDestinationInUseError, AlertForbiddenError, @@ -1572,6 +1573,24 @@ describe("AlertsService", () => { const userId = asUserId("user_timeout") const destination = yield* createWebhookDestination(alerts, orgId, userId) const rule = yield* createErrorRateRule(alerts, orgId, userId, destination.id) + const lifecycleEvent = projectAlertLifecycleEvent({ + tenantId: orgId, + ruleId: rule.id, + ruleName: rule.name, + incidentId: null, + eventType: "test", + incidentStatus: "resolved", + groupKey: null, + signalType: rule.signalType, + severity: rule.severity, + comparator: rule.comparator, + threshold: rule.threshold, + thresholdUpper: rule.thresholdUpper, + windowMinutes: rule.windowMinutes, + value: 0, + sampleCount: 0, + occurredAtMs: fixedTime, + }) yield* Effect.promise(() => insertDeliveryEventRow(testDb, { @@ -1586,6 +1605,7 @@ describe("AlertsService", () => { status: "queued", scheduledAt: fixedTime - 1, payloadJson: JSON.stringify({ + event: lifecycleEvent, eventType: "test", incidentId: null, incidentStatus: "resolved", @@ -1606,6 +1626,7 @@ describe("AlertsService", () => { }, linkUrl: "http://127.0.0.1:3471/alerts", sentAt: new Date(fixedTime).toISOString(), + futureAdditiveField: { preserve: true }, }), }), ) @@ -1614,6 +1635,18 @@ describe("AlertsService", () => { // live runtime clock, so the timeout fires on its own in real time. const tick = yield* alerts.runSchedulerTick() const events = yield* alerts.listDeliveryEvents(orgId) + const retryPayload = yield* Effect.promise(() => + queryFirstRow<{ + payload_json: { + event?: unknown + futureAdditiveField?: unknown + } + }>( + testDb, + "select payload_json from alert_delivery_events where delivery_key = $1 and attempt_number = 2", + ["timeout-delivery-key"], + ), + ) assert.strictEqual(tick.processedCount, 1) assert.strictEqual(tick.deliveryFailureCount, 1) @@ -1626,6 +1659,8 @@ describe("AlertsService", () => { assert.strictEqual(timeoutEvent?.status, "failed") assert.include(timeoutEvent?.errorMessage ?? "", "timed out") assert.strictEqual(retryEvent?.status, "queued") + assert.deepStrictEqual(retryPayload?.payload_json.event, lifecycleEvent) + assert.deepStrictEqual(retryPayload?.payload_json.futureAdditiveField, { preserve: true }) }).pipe( Effect.provide( makeLayer(testDb, makeWarehouseStub({ tracesAggregateRows: emptyWarehouseRows }), { diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 2404ac4d6..e18a366e9 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -9,6 +9,7 @@ import { type AlertLifecycleInput, } from "@maple/alerting-core" import { formatWarehouseDateTime } from "@maple/query-engine" +import { MapleCloudEventSchema } from "@maple/eventing-core" import { AlertComparator as AlertComparatorSchema, AlertDeliveryError, @@ -158,6 +159,7 @@ type DatabaseExecutor = DatabaseClient | DatabaseTransaction /* -------------------------------------------------------------------------- */ const StoredDeliveryPayloadSchema = Schema.Struct({ + event: Schema.optionalKey(MapleCloudEventSchema), eventType: Schema.optionalKey(Schema.String), incidentId: Schema.optionalKey(Schema.NullOr(Schema.String)), incidentStatus: Schema.optionalKey(Schema.String), @@ -1418,9 +1420,12 @@ export class AlertsService extends Context.Service row.payloadJson as Record), Effect.orElseSucceed(() => ({})), - )) as Record + ) yield* insertDeliveryEvent( row.orgId, row.incidentId, diff --git a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts index 5f6674ab1..0b4af9bd0 100644 --- a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts +++ b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts @@ -7,16 +7,30 @@ import { PlanetScaleWebhookPayload } from "./webhook-events" const QUEUE_BINDING = "PLANETSCALE_WEBHOOK_QUEUE" -export const PlanetScaleWebhookJob = Schema.Struct({ +const PlanetScaleWebhookJobFields = { kind: Schema.Literal("planetscale-webhook"), orgId: OrgId, connectionId: Schema.String, payload: PlanetScaleWebhookPayload, receivedAt: Schema.Number, +} as const + +/** Exact queue body emitted before the typed CloudEvent migration. */ +export const LegacyPlanetScaleWebhookJob = Schema.Struct(PlanetScaleWebhookJobFields) + +/** Current producer contract. New writers must always include the event. */ +export const PlanetScaleWebhookJob = Schema.Struct({ + ...PlanetScaleWebhookJobFields, event: MapleCloudEventSchema, }) export type PlanetScaleWebhookJob = Schema.Schema.Type +/** Consumer contract kept backward-compatible during rolling deployments. */ +export const PlanetScaleWebhookQueueMessage = Schema.Struct({ + ...PlanetScaleWebhookJobFields, + event: Schema.optionalKey(MapleCloudEventSchema), +}) + export class PlanetScaleWebhookQueueError extends Data.TaggedError( "@maple/api/services/planetscale/PlanetScaleWebhookQueueError", )<{ diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts index df8780ef7..99f17d175 100644 --- a/apps/cli/src/server/eventing/control-store.ts +++ b/apps/cli/src/server/eventing/control-store.ts @@ -5,8 +5,8 @@ import { pathToFileURL } from "node:url" import { canonicalJson, isJsonValue, - MapleCloudEventSchema, SignalProjectionSpecSchema, + validateMapleCloudEvent, type MapleCloudEvent, type JsonValue, type ProjectionFailure, @@ -18,8 +18,9 @@ import { durableWrite, ensurePrivateDirectory } from "../durable-files" const CONTROL_SCHEMA_VERSION = 1 const CONTROL_DIRECTORY = "control" const CONTROL_DATABASE = "eventing.sqlite" -const MAX_EVENT_BYTES = 256 * 1024 const MAX_FAILURES_PER_TENANT = 10_000 +export const DEFAULT_MAX_OUTBOX_EVENTS = 10_000 +export const DEFAULT_MAX_OUTBOX_BYTES = 256 * 1024 * 1024 export const eventingControlDirectory = (dataDir: string): string => join(resolve(dataDir), CONTROL_DIRECTORY) export const eventingControlPath = (dataDir: string): string => @@ -99,7 +100,10 @@ interface EventRow { } interface EventJsonRow { + readonly sequence: number | bigint readonly event_json: string + readonly staged_at: string + readonly ready_at: string | null } interface CountRow { @@ -110,6 +114,17 @@ interface QuickCheckRow { readonly quick_check: string } +interface WalCheckpointRow { + readonly busy: number | bigint + readonly log: number | bigint + readonly checkpointed: number | bigint +} + +interface OutboxUsageRow { + readonly count: number | bigint + readonly bytes: number | bigint +} + export interface StageEventsResult { readonly inserted: number readonly deduplicated: number @@ -124,6 +139,23 @@ export interface EventingControlSnapshotValidation { readonly readyEvents: number } +export interface LocalEventingControlLimits { + readonly maxOutboxEvents: number + readonly maxOutboxBytes: number +} + +export interface EventingOutboxRecord { + readonly sequence: number + readonly event: MapleCloudEvent + readonly stagedAt: string + readonly readyAt: string | null +} + +export interface EventingOutboxPage { + readonly events: readonly EventingOutboxRecord[] + readonly nextCursor: number | null +} + const asNumber = (value: number | bigint): number => { const number = Number(value) if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) @@ -134,9 +166,7 @@ const decodeProjection = (json: string): SignalProjectionSpec => Schema.decodeUnknownSync(SignalProjectionSpecSchema)(JSON.parse(json) as unknown) const decodeEvent = (json: string): MapleCloudEvent => { - const value = Schema.decodeUnknownSync(MapleCloudEventSchema)(JSON.parse(json) as unknown) - if (!isJsonValue(value.data)) throw new Error("stored CloudEvent data is not finite JSON") - return value as MapleCloudEvent + return validateMapleCloudEvent(JSON.parse(json) as unknown).event } const assertRealDatabaseFile = (path: string): void => { @@ -157,6 +187,26 @@ const configure = (db: Database): void => { db.exec("PRAGMA busy_timeout = 5000") } +const checkpointWal = (db: Database): void => { + const result = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get() + if (!result) throw new Error("eventing control WAL checkpoint returned no result") + const busy = asNumber(result.busy) + const log = asNumber(result.log) + const checkpointed = asNumber(result.checkpointed) + if (busy !== 0 || log !== 0) + throw new Error( + `eventing control WAL checkpoint incomplete (busy=${busy}, log=${log}, checkpointed=${checkpointed})`, + ) +} + +const validateLimits = (limits: LocalEventingControlLimits): LocalEventingControlLimits => { + if (!Number.isSafeInteger(limits.maxOutboxEvents) || limits.maxOutboxEvents < 1) + throw new Error("maxOutboxEvents must be a positive safe integer") + if (!Number.isSafeInteger(limits.maxOutboxBytes) || limits.maxOutboxBytes < 1) + throw new Error("maxOutboxBytes must be a positive safe integer") + return limits +} + const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation => { const quick = db.query("PRAGMA quick_check").get() if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) @@ -187,14 +237,23 @@ const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation = export class LocalEventingControlStore { readonly #db: Database + readonly #limits: LocalEventingControlLimits readonly path: string - private constructor(path: string, db: Database) { + private constructor(path: string, db: Database, limits: LocalEventingControlLimits) { this.path = path this.#db = db + this.#limits = limits } - static async open(dataDir: string): Promise { + static async open( + dataDir: string, + limits: LocalEventingControlLimits = { + maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, + maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, + }, + ): Promise { + validateLimits(limits) const directory = eventingControlDirectory(dataDir) await ensurePrivateDirectory(directory) const path = eventingControlPath(dataDir) @@ -214,7 +273,7 @@ export class LocalEventingControlStore { ) chmodSync(path, 0o600) validateOpenDatabase(db) - return new LocalEventingControlStore(path, db) + return new LocalEventingControlStore(path, db, limits) } catch (error) { db.close() throw error @@ -222,7 +281,7 @@ export class LocalEventingControlStore { } close(): void { - this.#db.exec("PRAGMA wal_checkpoint(TRUNCATE)") + checkpointWal(this.#db) this.#db.close(true) } @@ -302,12 +361,17 @@ export class LocalEventingControlStore { const eventIds: string[] = [] this.#db .transaction(() => { + const usage = this.#db + .query( + "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", + ) + .get() + if (!usage) throw new Error("event outbox usage query returned no row") + let outboxEvents = asNumber(usage.count) + let outboxBytes = asNumber(usage.bytes) for (const candidate of events) { - const event = Schema.decodeUnknownSync(MapleCloudEventSchema)(candidate) - if (!isJsonValue(event as unknown)) throw new Error("CloudEvent must be finite JSON") - const eventJson = canonicalJson(event as unknown as JsonValue) - if (Buffer.byteLength(eventJson, "utf8") > MAX_EVENT_BYTES) - throw new Error(`CloudEvent exceeds ${MAX_EVENT_BYTES} UTF-8 bytes`) + const validated = validateMapleCloudEvent(candidate) + const { event, canonicalJson: eventJson, byteLength: eventBytes } = validated const existing = this.#db .query( "SELECT event_id, event_json, state FROM outbox_events WHERE event_id = ?", @@ -318,6 +382,13 @@ export class LocalEventingControlStore { throw new Error(`event ID collision with different payload: ${event.id}`) deduplicated += 1 } else { + if ( + outboxEvents + 1 > this.#limits.maxOutboxEvents || + outboxBytes + eventBytes > this.#limits.maxOutboxBytes + ) + throw new Error( + `event outbox capacity exceeded (${outboxEvents}/${this.#limits.maxOutboxEvents} events, ${outboxBytes}/${this.#limits.maxOutboxBytes} bytes)`, + ) this.#db.run( "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, state, event_json, staged_at) VALUES (?, ?, ?, ?, 'staged', ?, ?)", [ @@ -330,6 +401,8 @@ export class LocalEventingControlStore { ], ) inserted += 1 + outboxEvents += 1 + outboxBytes += eventBytes } eventIds.push(event.id) } @@ -358,26 +431,53 @@ export class LocalEventingControlStore { .immediate() } - listReady(limit = 100): readonly MapleCloudEvent[] { + #listOutbox(state: "ready" | "staged", limit = 100, after = 0): EventingOutboxPage { if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) - throw new Error("ready-event limit must be between 1 and 1000") - return this.#db - .query( - "SELECT event_json FROM outbox_events WHERE state = 'ready' ORDER BY sequence LIMIT ?", + throw new Error("outbox-event limit must be between 1 and 1000") + if (!Number.isSafeInteger(after) || after < 0) + throw new Error("outbox cursor must be a non-negative safe integer") + const rows = this.#db + .query( + "SELECT sequence, event_json, staged_at, ready_at FROM outbox_events WHERE state = ? AND sequence > ? ORDER BY sequence LIMIT ?", ) - .all(limit) - .map(({ event_json }) => decodeEvent(event_json)) + .all(state, after, limit + 1) + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })) + return { + events: page, + nextCursor: hasMore ? (page.at(-1)?.sequence ?? null) : null, + } } - listStaged(limit = 100): readonly MapleCloudEvent[] { - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) - throw new Error("staged-event limit must be between 1 and 1000") - return this.#db - .query( - "SELECT event_json FROM outbox_events WHERE state = 'staged' ORDER BY sequence LIMIT ?", + listReady(limit = 100, after = 0): EventingOutboxPage { + return this.#listOutbox("ready", limit, after) + } + + listStaged(limit = 100, after = 0): EventingOutboxPage { + return this.#listOutbox("staged", limit, after) + } + + outboxCapacity(): LocalEventingControlLimits & { + readonly currentEvents: number + readonly currentBytes: number + } { + const usage = this.#db + .query( + "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", ) - .all(limit) - .map(({ event_json }) => decodeEvent(event_json)) + .get() + if (!usage) throw new Error("event outbox usage query returned no row") + return { + ...this.#limits, + currentEvents: asNumber(usage.count), + currentBytes: asNumber(usage.bytes), + } } recordProjectionFailures( @@ -412,6 +512,10 @@ export class LocalEventingControlStore { } async backupTo(path: string): Promise { + // sqlite3_serialize() snapshots the main database file. In WAL mode a + // committed transaction may still live only in the sidecar, so force and + // verify a complete checkpoint before copying the file image. + checkpointWal(this.#db) const bytes = this.#db.serialize() await durableWrite(path, bytes) return LocalEventingControlStore.validateSnapshot(path) diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts index b93b4659c..618b1bd87 100644 --- a/apps/cli/src/server/eventing/otlp.ts +++ b/apps/cli/src/server/eventing/otlp.ts @@ -231,9 +231,11 @@ const boundedIdentity = (value: string, prefix: string): string => : `${prefix}:sha256:${createHash("sha256").update(value, "utf8").digest("hex")}` const sourceUri = (resource: NormalizedAttributes, record: NormalizedAttributes): string => { - const explicit = stringAttribute(record, "event.source") ?? stringAttribute(record, "cloudevents.source") + const explicit = ( + stringAttribute(record, "event.source") ?? stringAttribute(record, "cloudevents.source") + )?.trim() if (explicit) return boundedIdentity(assertStringBound(explicit, "event source"), "urn:maple:source") - const service = stringAttribute(resource, "service.name") + const service = stringAttribute(resource, "service.name")?.trim() const source = service ? `urn:maple:source:otel:${encodeURIComponent(service)}` : "urn:maple:source:otel:local" @@ -241,11 +243,11 @@ const sourceUri = (resource: NormalizedAttributes, record: NormalizedAttributes) } const sourceOccurrenceId = (record: NormalizedAttributes): string | null => { - const value = - stringAttribute(record, "event.id") ?? - stringAttribute(record, "cloudevents.id") ?? - stringAttribute(record, "gitlab.event.id") - return value === null ? null : boundedIdentity(value, "source") + for (const key of ["event.id", "cloudevents.id", "gitlab.event.id"]) { + const value = stringAttribute(record, key)?.trim() + if (value) return boundedIdentity(value, "source") + } + return null } const derivedOccurrenceId = (input: JsonValue): string => @@ -301,7 +303,7 @@ export const normalizeOtlpLogs = ( occurrenceId: occurrenceId ?? derivedOccurrenceId({ source, occurredAt, signalKind: "otel.log", data }), - identityQuality: occurrenceId ? "source" : "derived", + identityQuality: occurrenceId === null ? "derived" : "source", occurredAt, observedAt: acceptedAt, subject, diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts index 2655c5865..fcefd51b9 100644 --- a/apps/cli/src/server/eventing/runtime.ts +++ b/apps/cli/src/server/eventing/runtime.ts @@ -2,6 +2,7 @@ import { CompiledProjectionRegistry, ProjectorRegistry, SignalSourceRegistry, + assertSignalProjectionInputBudget, fieldKey, isJsonValue, SignalProjectionSpecSchema, @@ -120,6 +121,13 @@ export interface LocalProjectionEvaluation { readonly typeMismatchFields: readonly string[] } +export interface LocalProjectionActivation { + readonly spec: SignalProjectionSpec + readonly next: readonly SignalProjectionSpec[] + readonly compiled: CompiledProjectionRegistry + readonly generation: number +} + const emptyEvaluation = (): LocalProjectionEvaluation => ({ events: [], failures: [], @@ -132,6 +140,7 @@ export class LocalEventingRuntime { readonly #projectors: ProjectorRegistry #compiled: CompiledProjectionRegistry #activeSourceKinds = new Set() + #generation = 0 constructor(store: LocalEventingControlStore) { this.#store = store @@ -146,7 +155,8 @@ export class LocalEventingRuntime { return this.#activeSourceKinds.has(sourceKind) } - activate(candidate: unknown): void { + prepareActivation(candidate: unknown): LocalProjectionActivation { + assertSignalProjectionInputBudget(candidate) const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) if (spec.tenantId !== TENANT_ID) throw new Error(`Maple Local only accepts projections for tenant ${TENANT_ID}`) @@ -155,9 +165,20 @@ export class LocalEventingRuntime { .filter((candidate) => candidate.id !== spec.id) const next = spec.enabled ? [...active, spec] : active const compiled = CompiledProjectionRegistry.compile(next, this.#sources, this.#projectors) - this.#store.saveProjection(spec) - this.#compiled = compiled - this.#activeSourceKinds = new Set(next.map(({ sourceKind }) => sourceKind)) + return { spec, next, compiled, generation: this.#generation } + } + + commitActivation(activation: LocalProjectionActivation): void { + if (activation.generation !== this.#generation) + throw new Error("projection registry changed during activation; retry the request") + this.#store.saveProjection(activation.spec) + this.#compiled = activation.compiled + this.#activeSourceKinds = new Set(activation.next.map(({ sourceKind }) => sourceKind)) + this.#generation += 1 + } + + activate(candidate: unknown): void { + this.commitActivation(this.prepareActivation(candidate)) } listActive(): readonly SignalProjectionSpec[] { @@ -200,17 +221,18 @@ export class LocalEventingRuntime { this.#store.markReady(eventIds) } - listReady(limit?: number): readonly MapleCloudEvent[] { - return this.#store.listReady(limit) + listReady(limit?: number, after?: number) { + return this.#store.listReady(limit, after) } - listStaged(limit?: number): readonly MapleCloudEvent[] { - return this.#store.listStaged(limit) + listStaged(limit?: number, after?: number) { + return this.#store.listStaged(limit, after) } health() { return { activeProjections: this.listActive().length, + outboxCapacity: this.#store.outboxCapacity(), ...this.#store.validate(), } } diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 6d06ed0c9..cdc47aacd 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -506,7 +506,7 @@ export class RequestQuiescenceGate { } async exclusive(work: () => Promise): Promise { - if (this.#closed) throw new Error("another server maintenance operation is active") + if (this.#closed) throw new MaintenanceInProgressError() this.#closed = true try { if (this.#active > 0) await new Promise((resolve) => this.#drained.push(resolve)) @@ -517,6 +517,57 @@ export class RequestQuiescenceGate { } } +class MaintenanceInProgressError extends Error { + constructor() { + super("another server maintenance operation is active") + this.name = "MaintenanceInProgressError" + } +} + +class RequestBodyTooLargeError extends Error { + constructor(readonly maximumBytes: number) { + super(`request body exceeds ${maximumBytes} bytes`) + this.name = "RequestBodyTooLargeError" + } +} + +const readBoundedJson = async (req: Request, maximumBytes: number): Promise => { + const contentLength = req.headers.get("content-length") + if (contentLength !== null && /^[0-9]+$/.test(contentLength)) { + const declared = Number(contentLength) + if (!Number.isSafeInteger(declared) || declared > maximumBytes) + throw new RequestBodyTooLargeError(maximumBytes) + } + if (req.body === null) return JSON.parse("") as unknown + const reader = req.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > maximumBytes) { + await reader.cancel() + throw new RequestBodyTooLargeError(maximumBytes) + } + chunks.push(value) + } + } finally { + reader.releaseLock() + } + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return JSON.parse(new TextDecoder().decode(bytes)) as unknown +} + +const invalidJsonResponse = (error: unknown): Response => + error instanceof RequestBodyTooLargeError ? text(error.message, 413) : text("invalid JSON body", 400) + const admitted = async (gate: RequestQuiescenceGate, work: () => Promise): Promise => { const leave = gate.enter() if (!leave) return text("server maintenance in progress", 503) @@ -569,12 +620,15 @@ const handleRetirement = async ( } const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i +const MAX_CHECKPOINT_BODY_BYTES = 4 * 1024 +const MAX_PROJECTION_BODY_BYTES = 512 * 1024 /** Typed, authenticated replacement for sending BACKUP through /local/query. */ const handleCheckpointBackup = async ( db: Chdb, controlStore: LocalEventingControlStore, dataDir: string, + gate: RequestQuiescenceGate, token: string, req: Request, ): Promise => { @@ -582,9 +636,9 @@ const handleCheckpointBackup = async ( return text("maintenance authorization required", 403) let body: unknown try { - body = await req.json() - } catch { - return text("invalid JSON body", 400) + body = await readBoundedJson(req, MAX_CHECKPOINT_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) } if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) const record = body as Record @@ -593,10 +647,13 @@ const handleCheckpointBackup = async ( if (!CHECKPOINT_ID.test(record.checkpointId)) return text("invalid checkpoint ID", 400) try { const checkpointId = record.checkpointId.toLowerCase() - const control = await controlStore.backupTo(eventingControlSnapshotPath(dataDir, checkpointId)) - db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) - return json({ checkpointId, control }) + return await gate.exclusive(async () => { + const control = await controlStore.backupTo(eventingControlSnapshotPath(dataDir, checkpointId)) + db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) + return json({ checkpointId, control }) + }) } catch (error) { + if (error instanceof MaintenanceInProgressError) return text(error.message, 409) return text( `checkpoint backup failed: ${error instanceof Error ? error.message : String(error)}`, 400, @@ -611,6 +668,7 @@ const eventingAuthorized = (token: string, req: Request): Response | null => const handleProjectionActivation = async ( eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, token: string, req: Request, ): Promise => { @@ -618,14 +676,26 @@ const handleProjectionActivation = async ( if (unauthorized) return unauthorized let body: unknown try { - body = await req.json() - } catch { - return text("invalid JSON body", 400) + body = await readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + let activation + try { + // Recursive schema validation and full registry compilation happen while + // normal ingest/query admission remains open. + activation = eventing.prepareActivation(body) + } catch (error) { + return text( + `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, + 400, + ) } try { - eventing.activate(body) + await gate.exclusive(async () => eventing.commitActivation(activation)) return json({ active: eventing.listActive() }) } catch (error) { + if (error instanceof MaintenanceInProgressError) return text(error.message, 409) return text( `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, 400, @@ -646,10 +716,12 @@ const handleEventingRead = ( if (url.pathname === "/local/eventing/outbox") { const rawLimit = url.searchParams.get("limit") const limit = rawLimit === null ? 100 : Number(rawLimit) + const rawAfter = url.searchParams.get("after") + const after = rawAfter === null ? 0 : Number(rawAfter) const state = url.searchParams.get("state") ?? "ready" try { - if (state === "ready") return json(eventing.listReady(limit)) - if (state === "staged") return json(eventing.listStaged(limit)) + if (state === "ready") return json(eventing.listReady(limit, after)) + if (state === "staged") return json(eventing.listStaged(limit, after)) return text("outbox state must be ready or staged", 400) } catch (error) { return text(error instanceof Error ? error.message : String(error), 400) @@ -699,14 +771,17 @@ const makeFetch = return respond(await admitted(gate, () => querySpan(runSpan, db, authority, req))) if (url.pathname === "/local/checkpoint/backup") return respond( - await gate.exclusive(() => - handleCheckpointBackup(db, controlStore, options.dataDir, maintenanceToken, req), + await handleCheckpointBackup( + db, + controlStore, + options.dataDir, + gate, + maintenanceToken, + req, ), ) if (url.pathname === "/local/eventing/projections") - return respond( - await gate.exclusive(() => handleProjectionActivation(eventing, maintenanceToken, req)), - ) + return respond(await handleProjectionActivation(eventing, gate, maintenanceToken, req)) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } @@ -861,4 +936,12 @@ export const startServer = ( return { port: server.port ?? options.port } }) -export const __testables = { handleEventingRead, ingest, recordServerResponse } +export const __testables = { + handleCheckpointBackup, + handleEventingRead, + handleProjectionActivation, + ingest, + readBoundedJson, + recordServerResponse, + RequestQuiescenceGate, +} diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts index 2f592e866..9a7263613 100644 --- a/apps/cli/test/local-eventing-control-store.test.ts +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -1,5 +1,5 @@ -import { deepStrictEqual, rejects, strictEqual, throws } from "node:assert" -import { mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs" +import { deepStrictEqual, ok, rejects, strictEqual, throws } from "node:assert" +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "vitest" @@ -96,8 +96,11 @@ describe("LocalEventingControlStore", () => { throws(() => store.markReady(["unknown"]), /unknown event/) store.markReady([event().id]) store.markReady([event().id]) - deepStrictEqual(store.listStaged(), []) - deepStrictEqual(store.listReady(), [event()]) + deepStrictEqual(store.listStaged().events, []) + deepStrictEqual( + store.listReady().events.map(({ event }) => event), + [event()], + ) } finally { store.close() } @@ -121,7 +124,10 @@ describe("LocalEventingControlStore", () => { store = await LocalEventingControlStore.open(dataDir) deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) - deepStrictEqual(store.listReady(), [event()]) + deepStrictEqual( + store.listReady().events.map(({ event }) => event), + [event()], + ) const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") const validation = await store.backupTo(snapshot) deepStrictEqual(validation, { @@ -142,12 +148,75 @@ describe("LocalEventingControlStore", () => { const restoredStore = await LocalEventingControlStore.open(restored) try { deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) - deepStrictEqual(restoredStore.listReady(), [event()]) + deepStrictEqual( + restoredStore.listReady().events.map(({ event }) => event), + [event()], + ) } finally { restoredStore.close() } })) + it("checkpoints committed live WAL state before serializing", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + store.stageEvents([event()]) + store.markReady([event().id]) + const walPath = `${eventingControlPath(dataDir)}-wal` + ok(existsSync(walPath)) + ok(statSync(walPath).size > 0, "test requires uncheckpointed WAL frames") + + const snapshot = join(dataDir, "backups", "live-wal", "control.sqlite") + await store.backupTo(snapshot) + strictEqual(statSync(walPath).size, 0) + + const restored = join(dataDir, "restored-live-wal") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + const restoredStore = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual( + restoredStore.listReady().events.map(({ event }) => event), + [event()], + ) + } finally { + restoredStore.close() + } + } finally { + store.close() + } + })) + + it("paginates every ready event and applies fail-closed outbox capacity", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 2, + maxOutboxBytes: 1024 * 1024, + }) + try { + const second = event({ id: "event-2", data: { iid: 43, title: "Second" } }) + const third = event({ id: "event-3", data: { iid: 44, title: "Third" } }) + const staged = store.stageEvents([event(), second]) + store.markReady(staged.eventIds) + + const firstPage = store.listReady(1) + strictEqual(firstPage.events.length, 1) + strictEqual(firstPage.nextCursor, firstPage.events[0]?.sequence) + const secondPage = store.listReady(1, firstPage.nextCursor!) + deepStrictEqual( + [...firstPage.events, ...secondPage.events].map(({ event }) => event.id), + [event().id, second.id], + ) + strictEqual(secondPage.nextCursor, null) + deepStrictEqual(store.stageEvents([event()]).deduplicated, 1) + throws(() => store.stageEvents([third]), /outbox capacity exceeded/) + } finally { + store.close() + } + })) + it("refuses a symlink in place of the database", async () => withDataDir(async (dataDir) => { const controlPath = eventingControlPath(dataDir) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts index a0e4b347e..5b64badfe 100644 --- a/apps/cli/test/local-eventing-ingest.test.ts +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -1,4 +1,4 @@ -import { deepStrictEqual, strictEqual } from "node:assert" +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" import { describe, it } from "vitest" import { __testables } from "../src/server/serve" @@ -7,8 +7,11 @@ describe("Local eventing ingest seam", () => { const eventing = { health: () => ({ activeProjections: 1 }), listActive: () => [], - listReady: () => [{ id: "ready" }], - listStaged: () => [{ id: "staged" }], + listReady: () => ({ events: [{ sequence: 1, event: { id: "ready" } }], nextCursor: null }), + listStaged: (_limit: number, after: number) => ({ + events: [{ sequence: after + 1, event: { id: "staged" } }], + nextCursor: null, + }), } const unauthorized = __testables.handleEventingRead( eventing as never, @@ -18,7 +21,7 @@ describe("Local eventing ingest seam", () => { ) strictEqual(unauthorized.status, 403) - const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged", { + const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged&after=41", { headers: { "x-maple-maintenance-token": "maintenance-secret" }, }) const authorized = __testables.handleEventingRead( @@ -28,10 +31,113 @@ describe("Local eventing ingest seam", () => { new URL(request.url), ) strictEqual(authorized.status, 200) - deepStrictEqual(await authorized.json(), [{ id: "staged" }]) + deepStrictEqual(await authorized.json(), { + events: [{ sequence: 42, event: { id: "staged" } }], + nextCursor: null, + }) + }) + + it("authenticates and reads activation bodies before closing admission", async () => { + const gate = new __testables.RequestQuiescenceGate() + const neverClosed = new ReadableStream() + const unauthorized = await __testables.handleProjectionActivation( + {} as never, + gate, + "maintenance-secret", + { + headers: new Headers(), + body: neverClosed, + } as Request, + ) + strictEqual(unauthorized.status, 403) + const afterUnauthorized = gate.enter() + ok(afterUnauthorized, "invalid authorization must not close admission") + afterUnauthorized() + + const checkpointUnauthorized = await __testables.handleCheckpointBackup( + {} as never, + {} as never, + "/unused", + gate, + "maintenance-secret", + { headers: new Headers(), body: neverClosed } as Request, + ) + strictEqual(checkpointUnauthorized.status, 403) + const afterCheckpointUnauthorized = gate.enter() + ok(afterCheckpointUnauthorized, "checkpoint authorization must precede exclusivity") + afterCheckpointUnauthorized() + + let controller!: ReadableStreamDefaultController + const slowBody = new ReadableStream({ + start(value) { + controller = value + }, + }) + let committed = false + const pending = __testables.handleProjectionActivation( + { + prepareActivation: (body: unknown) => ({ body }), + commitActivation: () => { + committed = true + }, + listActive: () => [], + } as never, + gate, + "maintenance-secret", + { + headers: new Headers({ "x-maple-maintenance-token": "maintenance-secret" }), + body: slowBody, + } as Request, + ) + await Promise.resolve() + const whileReading = gate.enter() + ok(whileReading, "an incomplete request body must not close admission") + whileReading() + controller.enqueue(new TextEncoder().encode("{}")) + controller.close() + strictEqual((await pending).status, 200) + strictEqual(committed, true) + }) + + it("bounds activation bodies and reports concurrent maintenance intentionally", async () => { + const oversized = new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + body: "123456789", + }) + await rejects(() => __testables.readBoundedJson(oversized, 8), /exceeds 8 bytes/) + + const gate = new __testables.RequestQuiescenceGate() + let releaseMaintenance!: () => void + const maintenance = gate.exclusive( + () => + new Promise((resolve) => { + releaseMaintenance = resolve + }), + ) + await Promise.resolve() + const response = await __testables.handleProjectionActivation( + { + prepareActivation: () => ({}), + commitActivation: () => undefined, + listActive: () => [], + } as never, + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: "{}", + }), + ) + strictEqual(response.status, 409) + releaseMaintenance() + await maintenance }) - it("evaluates and stages before chDB write, then marks ready before acknowledging", async () => { + it("isolates projection failures, stores telemetry, and makes sibling events ready", async () => { const order: string[] = [] const event = { id: "event-1" } const db = { @@ -49,7 +155,18 @@ describe("Local eventing ingest seam", () => { const eventing = { evaluateOtlp: () => { order.push("evaluate") - return { events: [event], failures: [], typeMismatchFields: [] } + return { + events: [event], + failures: [ + { + projectionId: "oversized-projector", + projectionRevision: 1, + occurrenceId: "occurrence-1", + message: "CloudEvent exceeds 262144 UTF-8 bytes", + }, + ], + typeMismatchFields: [], + } }, persistFailures: () => order.push("persist-failures"), stage: () => { diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts index 30c1a7915..6a72646f6 100644 --- a/apps/cli/test/local-eventing-runtime.test.ts +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -64,6 +64,9 @@ const gitlabIssueCreated = { ], } +const firstLogRecord = (request: typeof gitlabIssueCreated) => + request.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]! + const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ id: "gitlab-issue-created", revision: 1, @@ -106,6 +109,38 @@ describe("LocalEventingRuntime", () => { strictEqual(JSON.parse(batches[0]!.ndjson).log_attributes["gitlab.issue.iid"], "42") }) + it("uses the first nonblank occurrence alias and derives identity when every alias is blank", () => { + const aliased = structuredClone(gitlabIssueCreated) + const aliasedRecord = firstLogRecord(aliased) + aliasedRecord.attributes = [ + attr("event.id", { stringValue: " " }), + attr("cloudevents.id", { stringValue: " cloud-event-42 " }), + attr("gitlab.event.id", { stringValue: "gitlab-event-42" }), + ...aliasedRecord.attributes.filter( + ({ key }) => !["event.id", "cloudevents.id", "gitlab.event.id"].includes(key), + ), + ] + const [aliasedSignal] = normalizeOtlpLogs(aliased, "2026-08-07T20:00:00Z") + strictEqual(aliasedSignal?.occurrenceId, "cloud-event-42") + strictEqual(aliasedSignal?.identityQuality, "source") + + const derivedA = structuredClone(aliased) + const derivedARecord = firstLogRecord(derivedA) + derivedARecord.attributes = derivedARecord.attributes.map((entry) => + ["event.id", "cloudevents.id", "gitlab.event.id"].includes(entry.key) + ? attr(entry.key, { stringValue: entry.key === "event.id" ? "" : " \t " }) + : entry, + ) + const derivedB = structuredClone(derivedA) + firstLogRecord(derivedB).body = { stringValue: "A different issue occurrence" } + const [signalA] = normalizeOtlpLogs(derivedA, "2026-08-07T20:00:00Z") + const [signalB] = normalizeOtlpLogs(derivedB, "2026-08-07T20:00:00Z") + strictEqual(signalA?.identityQuality, "derived") + strictEqual(signalB?.identityQuality, "derived") + strictEqual(signalA?.occurrenceId?.startsWith("derived:sha256:"), true) + strictEqual(signalA?.occurrenceId === signalB?.occurrenceId, false) + }) + it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => withDataDir(async (dataDir) => { const store = await LocalEventingControlStore.open(dataDir) @@ -144,14 +179,20 @@ describe("LocalEventingRuntime", () => { }) const staged = runtime.stage(first.events) strictEqual(staged.inserted, 1) - strictEqual(runtime.listReady().length, 0) - deepStrictEqual(runtime.listStaged(), first.events) + strictEqual(runtime.listReady().events.length, 0) + deepStrictEqual( + runtime.listStaged().events.map(({ event }) => event), + first.events, + ) const retry = runtime.evaluateOtlp("logs", gitlabIssueCreated) strictEqual(retry.events[0]?.id, first.events[0]?.id) strictEqual(runtime.stage(retry.events).deduplicated, 1) runtime.markReady(staged.eventIds) - deepStrictEqual(runtime.listReady(), first.events) - deepStrictEqual(runtime.listStaged(), []) + deepStrictEqual( + runtime.listReady().events.map(({ event }) => event), + first.events, + ) + deepStrictEqual(runtime.listStaged().events, []) } finally { store.close() } diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md index b1e1f91ce..e9e59cec7 100644 --- a/docs/signal-to-event-projection.md +++ b/docs/signal-to-event-projection.md @@ -819,7 +819,10 @@ follows: - Maple Local stores projection revisions, failures, and the staged/ready outbox in SQLite at `/control/eventing.sqlite`, using WAL and `synchronous = -FULL`. A version-2 Maple checkpoint contains `control.sqlite` beside the chDB +FULL`. While ingest is quiesced, backup first completes and verifies a blocking + `wal_checkpoint(TRUNCATE)` so the serialized database contains every committed + control-store transaction rather than only the main SQLite file. A version-2 + Maple checkpoint contains `control.sqlite` beside the chDB backup and binds its byte count, SHA-256 digest, schema version, and row counts in the checkpoint manifest. Version-1 checkpoints remain readable and restore an empty control store. @@ -849,11 +852,16 @@ FULL`. A version-2 Maple checkpoint contains `control.sqlite` beside the chDB acknowledges them. The dedicated Cloudflare Queue durably carries `dev.maple.planetscale.webhook.received.v1`; its temporary provider payload keeps the existing issue and timeline consumers behaviorally unchanged while - they migrate to the event contract. + they migrate to the event contract. Queue consumers accept both the new + event-bearing message and the exact pre-migration message shape, reconstructing + the deterministic event from the older message's durable fields during rolling + upgrades. - Hosted query-alert delivery rows remain that producer's durable outbox. Their payload now includes an additive deterministic `dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` CloudEvent while - retaining every legacy top-level delivery field. + retaining every legacy top-level delivery field. Retry creation preserves the + originally stored JSON, including the CloudEvent ID and future additive fields, + instead of round-tripping it through a lossy legacy schema. - Historical replay execution remains deliberately unimplemented in this change. Field catalogs already declare `exact`, `coerced`, or `unavailable`, but Local's current arbitrary attribute maps have lost source scalar type and @@ -870,12 +878,23 @@ FULL`. A version-2 Maple checkpoint contains `control.sqlite` beside the chDB Maple Local activates immutable revisions with authenticated `POST /local/eventing/projections`. The same maintenance credential protects `GET /local/eventing/projections`, `/local/eventing/health`, and -`/local/eventing/outbox`. The outbox endpoint returns ready events by default; -`?state=staged` exposes bounded inspection of records stranded before the chDB -commit point. Re-delivery is the safe recovery operation: it deduplicates the -same staged event ID and promotes it only after the warehouse write succeeds. -Maple never blindly promotes an old staged record because, after a crash, the -control store alone cannot prove whether the corresponding chDB write committed. -Activation compiles the entire candidate registry -before the SQLite commit and swaps the immutable runtime snapshot while ingest -is quiesced, so a request observes exactly one registry version. +`/local/eventing/outbox`. The outbox endpoint returns ready records with their +monotonic `sequence` by default; `?after=&limit=` pages beyond the +oldest page, while `?state=staged` exposes bounded inspection of records stranded +before the chDB commit point. The Local store defaults to at most 10,000 events +and 256 MiB of canonical event JSON. Staging fails closed with a retryable ingest +error before either cap can be exceeded. These endpoints are an operable +inspection/recovery surface, not yet a delivery protocol: durable consumer +claim/acknowledgement and deletion/retention begin with the first downstream +consumer rather than being implied by a destructive read API in this PR. + +Re-delivery is the safe recovery operation: it deduplicates the same staged event +ID and promotes it only after the warehouse write succeeds. Maple never blindly +promotes an old staged record because, after a crash, the control store alone +cannot prove whether the corresponding chDB write committed. Activation requires +authentication, a bounded request body, structural budget validation, and full +registry compilation before acquiring global quiescence. Only the projection +revision commit and immutable runtime-registry swap occur while ingest is +quiesced, so invalid credentials, incomplete bodies, and expensive validation do +not close admission and every ingest request still observes exactly one registry +version. Concurrent maintenance requests receive an intentional conflict response. diff --git a/packages/eventing-core/schemas/signal-projection.v1.schema.json b/packages/eventing-core/schemas/signal-projection.v1.schema.json index 44f2adbae..f9d9b935f 100644 --- a/packages/eventing-core/schemas/signal-projection.v1.schema.json +++ b/packages/eventing-core/schemas/signal-projection.v1.schema.json @@ -39,7 +39,12 @@ "enum": ["string"] }, "value": { - "type": "string" + "type": "string", + "allOf": [ + { + "maxLength": 4096 + } + ] } }, "required": ["type", "value"], @@ -69,6 +74,9 @@ "value": { "type": "string", "allOf": [ + { + "maxLength": 20 + }, { "pattern": "^-?(?:0|[1-9][0-9]*)$" } @@ -121,6 +129,9 @@ "value": { "type": "string", "allOf": [ + { + "maxLength": 20 + }, { "pattern": "^-?(?:0|[1-9][0-9]*)$" } @@ -145,7 +156,15 @@ "type": "array", "items": { "$ref": "#/$defs/SignalPredicate" - } + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 64 + } + ] } }, "required": ["op", "clauses"], @@ -162,7 +181,15 @@ "type": "array", "items": { "$ref": "#/$defs/SignalPredicate" - } + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 64 + } + ] } }, "required": ["op", "clauses"], @@ -227,7 +254,15 @@ "type": "array", "items": { "$ref": "#/$defs/SignalScalar" - } + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 100 + } + ] } }, "required": ["op", "field", "values"], diff --git a/packages/eventing-core/schemas/signal-scalar.v1.schema.json b/packages/eventing-core/schemas/signal-scalar.v1.schema.json index bbf5af1c1..81fa5bf22 100644 --- a/packages/eventing-core/schemas/signal-scalar.v1.schema.json +++ b/packages/eventing-core/schemas/signal-scalar.v1.schema.json @@ -13,7 +13,12 @@ "enum": ["string"] }, "value": { - "type": "string" + "type": "string", + "allOf": [ + { + "maxLength": 4096 + } + ] } }, "required": ["type", "value"], @@ -43,6 +48,9 @@ "value": { "type": "string", "allOf": [ + { + "maxLength": 20 + }, { "pattern": "^-?(?:0|[1-9][0-9]*)$" } @@ -95,6 +103,9 @@ "value": { "type": "string", "allOf": [ + { + "maxLength": 20 + }, { "pattern": "^-?(?:0|[1-9][0-9]*)$" } diff --git a/packages/eventing-core/src/event.ts b/packages/eventing-core/src/event.ts index e70308e78..6dd7f42e3 100644 --- a/packages/eventing-core/src/event.ts +++ b/packages/eventing-core/src/event.ts @@ -1,7 +1,16 @@ import { createHash } from "node:crypto" -import type { JsonValue, MapleCloudEvent, NormalizedSignal, SignalProjectionSpec } from "./model" +import { Schema } from "effect" +import { + MapleCloudEventSchema, + type JsonValue, + type MapleCloudEvent, + type NormalizedSignal, + type SignalProjectionSpec, +} from "./model" import { timestampToEpochNanos } from "./predicate" +export const MAX_CLOUD_EVENT_BYTES = 256 * 1024 + export interface EventIdentityInput { readonly tenantId: string readonly sourceKind: string @@ -70,6 +79,23 @@ export const canonicalJson = (value: JsonValue): string => { return JSON.stringify(canonicalizeJson(value)) } +export interface ValidatedMapleCloudEvent { + readonly event: MapleCloudEvent + readonly canonicalJson: string + readonly byteLength: number +} + +/** Validate the complete persisted envelope, including its canonical byte budget. */ +export const validateMapleCloudEvent = (candidate: unknown): ValidatedMapleCloudEvent => { + const event = Schema.decodeUnknownSync(MapleCloudEventSchema)(candidate) + if (!isJsonValue(event as unknown)) throw new Error("CloudEvent must be finite JSON") + const eventJson = canonicalJson(event as unknown as JsonValue) + const byteLength = Buffer.byteLength(eventJson, "utf8") + if (byteLength > MAX_CLOUD_EVENT_BYTES) + throw new Error(`CloudEvent exceeds ${MAX_CLOUD_EVENT_BYTES} UTF-8 bytes`) + return { event: event as MapleCloudEvent, canonicalJson: eventJson, byteLength } +} + export const makeCloudEvent = (input: { readonly signal: NormalizedSignal readonly projection: SignalProjectionSpec @@ -81,17 +107,21 @@ export const makeCloudEvent = (input: { readonly time?: string readonly data: JsonValue }): MapleCloudEvent => { - if (input.signal.occurrenceId === null || input.signal.identityQuality === "none") + if ( + input.signal.occurrenceId === null || + input.signal.occurrenceId.trim().length === 0 || + input.signal.identityQuality === "none" + ) throw new Error("durable event projection requires stable or derived occurrence identity") if (!isJsonValue(input.data)) throw new Error("projected event data must be finite JSON") - if (input.outputType.length === 0) throw new Error("projected event type must not be empty") - if (input.dataSchema.length === 0) throw new Error("projected event data schema must not be empty") - if (input.signal.source.length === 0) throw new Error("signal source must not be empty") + if (input.outputType.trim().length === 0) throw new Error("projected event type must not be empty") + if (input.dataSchema.trim().length === 0) throw new Error("projected event data schema must not be empty") + if (input.signal.source.trim().length === 0) throw new Error("signal source must not be empty") const subject = input.subject ?? input.signal.subject const time = input.time ?? input.signal.occurredAt if (timestampToEpochNanos(time) === null) throw new Error("projected event time must be a valid instant") - return { + return validateMapleCloudEvent({ specversion: "1.0", id: makeEventId({ tenantId: input.signal.tenantId, @@ -113,5 +143,5 @@ export const makeCloudEvent = (input: { projectorid: input.projectorId, projectorversion: input.projectorVersion, data: input.data, - } + }).event } diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts index 9b291b5e4..48fd45dbb 100644 --- a/packages/eventing-core/src/model.ts +++ b/packages/eventing-core/src/model.ts @@ -1,12 +1,21 @@ import { Schema } from "effect" +export const MAX_PREDICATE_DEPTH = 8 +export const MAX_PREDICATE_NODES = 64 +export const MAX_IN_VALUES = 100 +export const MAX_STRING_LITERAL_BYTES = 4 * 1024 +export const MAX_DECIMAL_INT64_LENGTH = 20 + const NonEmptyIdentifier = Schema.String.check( Schema.isMinLength(1), Schema.isMaxLength(256), Schema.isTrimmed(), ) -const DecimalInt64 = Schema.String.check(Schema.isPattern(/^-?(?:0|[1-9][0-9]*)$/)) +const DecimalInt64 = Schema.String.check( + Schema.isMaxLength(MAX_DECIMAL_INT64_LENGTH), + Schema.isPattern(/^-?(?:0|[1-9][0-9]*)$/), +) const Rfc3339Timestamp = Schema.String.check( Schema.isPattern(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/), @@ -14,7 +23,7 @@ const Rfc3339Timestamp = Schema.String.check( export const StringSignalScalar = Schema.Struct({ type: Schema.Literal("string"), - value: Schema.String, + value: Schema.String.check(Schema.isMaxLength(MAX_STRING_LITERAL_BYTES)), }) export const BooleanSignalScalar = Schema.Struct({ @@ -108,11 +117,17 @@ export const SignalPredicateSchema: Schema.Codec, ).annotate({ identifier: "SignalPredicate" }) diff --git a/packages/eventing-core/src/predicate.test.ts b/packages/eventing-core/src/predicate.test.ts index c4cf98362..6c1b3495a 100644 --- a/packages/eventing-core/src/predicate.test.ts +++ b/packages/eventing-core/src/predicate.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from "node:fs" import { Schema } from "effect" import { describe, expect, it } from "vitest" import { + assertSignalProjectionInputBudget, compileSignalPredicate, defineSignalFields, fieldKey, @@ -120,6 +121,40 @@ describe("selector validation", () => { expect.objectContaining({ message: "int64 must be a signed 64-bit decimal integer" }), ) }) + + it("rejects hostile raw predicate topology before recursive schema decoding", () => { + let deeplyNested: unknown = { + op: "exists", + field: { namespace: "attribute", key: "x", type: "string" }, + } + for (let index = 0; index < MAX_PREDICATE_DEPTH; index++) + deeplyNested = { op: "not", clause: deeplyNested } + expect(() => assertSignalProjectionInputBudget({ selector: deeplyNested })).toThrow( + "predicate depth exceeds", + ) + + expect(() => + assertSignalProjectionInputBudget({ + selector: { + op: "all", + clauses: Array.from({ length: 65 }, () => ({ + op: "exists", + field: { namespace: "attribute", key: "x", type: "string" }, + })), + }, + }), + ).toThrow("clause list exceeds") + + expect(() => + assertSignalProjectionInputBudget({ + selector: { + op: "eq", + field: { namespace: "attribute", key: "n", type: "int64" }, + value: { type: "int64", value: "1".repeat(21) }, + }, + }), + ).toThrow("int64 literal exceeds") + }) }) describe("total runtime behavior", () => { diff --git a/packages/eventing-core/src/predicate.ts b/packages/eventing-core/src/predicate.ts index ede5adf13..850c0ca19 100644 --- a/packages/eventing-core/src/predicate.ts +++ b/packages/eventing-core/src/predicate.ts @@ -6,12 +6,14 @@ import type { SignalScalar, SignalScalarType, } from "./model" -import { fieldKey } from "./model" - -export const MAX_PREDICATE_DEPTH = 8 -export const MAX_PREDICATE_NODES = 64 -export const MAX_IN_VALUES = 100 -export const MAX_STRING_LITERAL_BYTES = 4 * 1024 +import { + fieldKey, + MAX_DECIMAL_INT64_LENGTH, + MAX_IN_VALUES, + MAX_PREDICATE_DEPTH, + MAX_PREDICATE_NODES, + MAX_STRING_LITERAL_BYTES, +} from "./model" const INT64_MIN = -(1n << 63n) const INT64_MAX = (1n << 63n) - 1n @@ -34,6 +36,72 @@ export class SignalPredicateValidationError extends Error { const stringBytes = (value: string): number => new TextEncoder().encode(value).byteLength +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const assertScalarInputBudget = (value: unknown): void => { + if (!isRecord(value) || typeof value.type !== "string") return + if (value.type === "string" && typeof value.value === "string") { + if (stringBytes(value.value) > MAX_STRING_LITERAL_BYTES) + throw new Error(`selector string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`) + return + } + if ( + (value.type === "int64" || value.type === "duration") && + typeof value.value === "string" && + value.value.length > MAX_DECIMAL_INT64_LENGTH + ) + throw new Error(`${value.type} literal exceeds ${MAX_DECIMAL_INT64_LENGTH} characters`) +} + +/** + * Reject hostile selector topology before the recursive runtime schema sees it. + * HTTP hosts should additionally bound the serialized request body. + */ +export const assertSignalProjectionInputBudget = (candidate: unknown): void => { + if (!isRecord(candidate) || candidate.selector === undefined) return + const stack: Array<{ readonly value: unknown; readonly depth: number }> = [ + { value: candidate.selector, depth: 1 }, + ] + const seen = new Set() + let nodes = 0 + while (stack.length > 0) { + const current = stack.pop()! + if (current.depth > MAX_PREDICATE_DEPTH) + throw new Error(`predicate depth exceeds ${MAX_PREDICATE_DEPTH}`) + nodes += 1 + if (nodes > MAX_PREDICATE_NODES) throw new Error(`predicate exceeds ${MAX_PREDICATE_NODES} nodes`) + if (!isRecord(current.value)) continue + if (seen.has(current.value)) throw new Error("predicate must be acyclic JSON") + seen.add(current.value) + + switch (current.value.op) { + case "all": + case "any": { + const clauses = current.value.clauses + if (!Array.isArray(clauses)) break + if (clauses.length > MAX_PREDICATE_NODES) + throw new Error(`predicate clause list exceeds ${MAX_PREDICATE_NODES} entries`) + for (let index = clauses.length - 1; index >= 0; index--) + stack.push({ value: clauses[index], depth: current.depth + 1 }) + break + } + case "not": + stack.push({ value: current.value.clause, depth: current.depth + 1 }) + break + case "in": { + const values = current.value.values + if (!Array.isArray(values)) break + if (values.length > MAX_IN_VALUES) throw new Error(`in exceeds ${MAX_IN_VALUES} values`) + for (const value of values) assertScalarInputBudget(value) + break + } + default: + assertScalarInputBudget(current.value.value) + } + } +} + const parseInt64 = (value: string): bigint | null => { try { const parsed = BigInt(value) diff --git a/packages/eventing-core/src/registry.test.ts b/packages/eventing-core/src/registry.test.ts index e7bfd3a9a..d85a7161e 100644 --- a/packages/eventing-core/src/registry.test.ts +++ b/packages/eventing-core/src/registry.test.ts @@ -4,6 +4,7 @@ import { canonicalJson, defineSignalFields, makeEventId, + MAX_CLOUD_EVENT_BYTES, ProjectorRegistry, SignalSourceRegistry, type NormalizedSignal, @@ -164,6 +165,56 @@ describe("CompiledProjectionRegistry", () => { ]) }) + it("isolates complete-envelope schema and size failures from successful siblings", () => { + const registryDefinitions = projectors() + .register({ + id: "oversized", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.oversized.v1", + dataSchema: "urn:maple:event-schema:oversized:v1", + decodeConfig: () => ({}), + project: () => ({ data: { payload: "x".repeat(MAX_CLOUD_EVENT_BYTES) } }), + }) + .register({ + id: "invalid-envelope", + version: 1, + sourceKinds: ["otel.log"], + outputType: "x".repeat(257), + dataSchema: "urn:maple:event-schema:invalid-envelope:v1", + decodeConfig: () => ({}), + project: () => ({ data: {} }), + }) + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ + id: "oversized-projection", + projector: { id: "oversized", version: 1, config: {} }, + }), + projection({ + id: "invalid-envelope-projection", + projector: { id: "invalid-envelope", version: 1, config: {} }, + }), + ], + sources(), + registryDefinitions, + ) + const result = registry.evaluate(signal()) + expect(result.events.map(({ projectionid }) => projectionid)).toEqual(["gitlab-issue-created"]) + expect(result.failures).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + projectionId: "oversized-projection", + message: expect.stringContaining("CloudEvent exceeds"), + }), + expect.objectContaining({ + projectionId: "invalid-envelope-projection", + }), + ]), + ) + }) + it("isolates tenants, source kinds, activation time, and disabled revisions", () => { const registry = CompiledProjectionRegistry.compile( [ diff --git a/packages/eventing-core/src/registry.ts b/packages/eventing-core/src/registry.ts index b5eb66adb..92c5bb7f0 100644 --- a/packages/eventing-core/src/registry.ts +++ b/packages/eventing-core/src/registry.ts @@ -31,8 +31,10 @@ export class ProjectorRegistry { if (!Number.isSafeInteger(projector.version) || projector.version < 1) throw new Error("projector version must be a positive safe integer") if (projector.sourceKinds.length === 0) throw new Error("projector must accept a source kind") - if (projector.outputType.length === 0) throw new Error("projector output type must not be empty") - if (projector.dataSchema.length === 0) throw new Error("projector data schema must not be empty") + if (projector.outputType.trim().length === 0) + throw new Error("projector output type must not be empty") + if (projector.dataSchema.trim().length === 0) + throw new Error("projector data schema must not be empty") const key = ProjectorRegistry.key(projector.id, projector.version) if (this.#projectors.has(key)) throw new Error(`duplicate projector registration: ${key}`) this.#projectors.set(key, projector as ErasedSignalProjector) From 0212b998416994ae939fef6ddf9f138780d4774a Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 11 Aug 2026 18:21:54 -0400 Subject: [PATCH 04/12] fix(eventing): stabilize outbox and selector contracts --- apps/api/src/services/alerts/AlertsService.ts | 1 - apps/cli/src/server/eventing/control-store.ts | 56 +++++++++++++++++-- apps/cli/src/server/eventing/otlp.ts | 12 ++-- .../test/local-eventing-control-store.test.ts | 29 ++++++++++ apps/cli/test/local-eventing-runtime.test.ts | 33 ++++++++++- apps/cli/test/server-network.test.ts | 3 +- docs/signal-to-event-projection.md | 39 ++++++++----- packages/eventing-core/fixtures/v1.json | 14 +++++ .../schemas/signal-projection.v1.schema.json | 12 ++-- .../schemas/signal-scalar.v1.schema.json | 7 +-- packages/eventing-core/src/model.ts | 36 ++++++++++-- packages/eventing-core/src/predicate.test.ts | 41 ++++++++++++++ packages/eventing-core/src/predicate.ts | 19 ++++--- packages/eventing-core/src/source.ts | 24 ++++++-- 14 files changed, 266 insertions(+), 60 deletions(-) diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index e18a366e9..039a12abb 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -33,7 +33,6 @@ import { AlertSignalType as AlertSignalTypeSchema, AlertValidationError, AlertNotificationTemplate, - type AlertComparator, type AlertDestinationType, type AlertEventType as AlertEventTypeValue, type AlertRuleUpsertRequest, diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts index 99f17d175..827a02d43 100644 --- a/apps/cli/src/server/eventing/control-store.ts +++ b/apps/cli/src/server/eventing/control-store.ts @@ -61,9 +61,18 @@ CREATE TABLE outbox_events ( ready_at TEXT ) STRICT; -CREATE INDEX outbox_events_ready_sequence +CREATE INDEX outbox_events_staged_sequence ON outbox_events (state, sequence); +CREATE TABLE outbox_ready_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + ready_at TEXT NOT NULL, + FOREIGN KEY (event_id) + REFERENCES outbox_events (event_id) + ON DELETE RESTRICT +) STRICT; + CREATE TABLE projection_failures ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id TEXT NOT NULL, @@ -226,6 +235,21 @@ const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation = if (!revisions) throw new Error("eventing projection count query returned no row") const failures = db.query("SELECT count(*) AS count FROM projection_failures").get() if (!failures) throw new Error("eventing projection-failure count query returned no row") + const invalidReadiness = db + .query( + `SELECT count(*) AS count + FROM outbox_events AS event + LEFT JOIN outbox_ready_events AS readiness ON readiness.event_id = event.event_id + WHERE (event.state = 'ready' AND ( + readiness.event_id IS NULL OR event.ready_at IS NULL OR event.ready_at <> readiness.ready_at + )) OR (event.state = 'staged' AND ( + readiness.event_id IS NOT NULL OR event.ready_at IS NOT NULL + ))`, + ) + .get() + if (!invalidReadiness) throw new Error("eventing readiness validation query returned no row") + if (asNumber(invalidReadiness.count) !== 0) + throw new Error("eventing control database has inconsistent outbox readiness state") return { schemaVersion, projectionRevisions: asNumber(revisions.count), @@ -422,6 +446,10 @@ export class LocalEventingControlStore { .get(eventId) if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) if (row.state === "ready") continue + this.#db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ + eventId, + readyAt, + ]) this.#db.run( "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", [readyAt, eventId], @@ -436,11 +464,27 @@ export class LocalEventingControlStore { throw new Error("outbox-event limit must be between 1 and 1000") if (!Number.isSafeInteger(after) || after < 0) throw new Error("outbox cursor must be a non-negative safe integer") - const rows = this.#db - .query( - "SELECT sequence, event_json, staged_at, ready_at FROM outbox_events WHERE state = ? AND sequence > ? ORDER BY sequence LIMIT ?", - ) - .all(state, after, limit + 1) + const rows = + state === "ready" + ? this.#db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(after, limit + 1) + : this.#db + .query( + `SELECT sequence, event_json, staged_at, ready_at + FROM outbox_events + WHERE state = 'staged' AND sequence > ? + ORDER BY sequence + LIMIT ?`, + ) + .all(after, limit + 1) const hasMore = rows.length > limit const pageRows = hasMore ? rows.slice(0, limit) : rows const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts index 618b1bd87..964665a68 100644 --- a/apps/cli/src/server/eventing/otlp.ts +++ b/apps/cli/src/server/eventing/otlp.ts @@ -82,31 +82,31 @@ export const OTLP_LOG_SOURCE: SignalSourceDefinition = { catalog("span.id", "string", equalityOperators), catalog("time", "timestamp"), catalog("observed_time", "timestamp"), - ], - openFields: [ { - namespace: "resource", + field: { namespace: "body", key: "value" }, types: ["string", "boolean", "int64", "float64"], operators: allOperators, sensitivity: "public", replay: "coerced", }, + ], + openFields: [ { - namespace: "scope", + namespace: "resource", types: ["string", "boolean", "int64", "float64"], operators: allOperators, sensitivity: "public", replay: "coerced", }, { - namespace: "attribute", + namespace: "scope", types: ["string", "boolean", "int64", "float64"], operators: allOperators, sensitivity: "public", replay: "coerced", }, { - namespace: "body", + namespace: "attribute", types: ["string", "boolean", "int64", "float64"], operators: allOperators, sensitivity: "public", diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts index 9a7263613..adad8ad36 100644 --- a/apps/cli/test/local-eventing-control-store.test.ts +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -217,6 +217,35 @@ describe("LocalEventingControlStore", () => { } })) + it("pages recovered events by first readiness transition instead of staging order", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const first = event({ id: "event-a" }) + const second = event({ id: "event-b" }) + store.stageEvents([first]) + store.stageEvents([second]) + store.markReady([second.id]) + + const initialPage = store.listReady(1) + deepStrictEqual( + initialPage.events.map(({ event }) => event.id), + [second.id], + ) + const cursor = initialPage.events[0]!.sequence + + store.markReady([first.id]) + const recoveredPage = store.listReady(1, cursor) + deepStrictEqual( + recoveredPage.events.map(({ event }) => event.id), + [first.id], + ) + strictEqual(recoveredPage.events[0]!.sequence > cursor, true) + } finally { + store.close() + } + })) + it("refuses a symlink in place of the database", async () => withDataDir(async (dataDir) => { const controlPath = eventingControlPath(dataDir) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts index 6a72646f6..a68559ea2 100644 --- a/apps/cli/test/local-eventing-runtime.test.ts +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -1,4 +1,4 @@ -import { deepStrictEqual, strictEqual } from "node:assert" +import { deepStrictEqual, strictEqual, throws } from "node:assert" import { mkdirSync, mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" @@ -141,6 +141,37 @@ describe("LocalEventingRuntime", () => { strictEqual(signalA?.occurrenceId === signalB?.occurrenceId, false) }) + it("catalogs only the scalar body field that the OTLP adapter can populate", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + throws( + () => + runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "text", type: "string" }, + }, + }), + ), + /unknown field body:text/, + ) + const activation = runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "value", type: "boolean" }, + }, + }), + ) + strictEqual(activation.spec.selector.op, "exists") + } finally { + store.close() + } + })) + it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => withDataDir(async (dataDir) => { const store = await LocalEventingControlStore.open(dataDir) diff --git a/apps/cli/test/server-network.test.ts b/apps/cli/test/server-network.test.ts index 46f389d8a..02b295ccf 100644 --- a/apps/cli/test/server-network.test.ts +++ b/apps/cli/test/server-network.test.ts @@ -171,8 +171,7 @@ describe("browser origin policy", () => { deepStrictEqual(corsHeadersForAllowedOrigin(hostedOrigin), { "access-control-allow-origin": hostedOrigin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": - "content-type, content-encoding, x-maple-maintenance-token", + "access-control-allow-headers": "content-type, content-encoding, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", }) diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md index e9e59cec7..2e7fb7478 100644 --- a/docs/signal-to-event-projection.md +++ b/docs/signal-to-event-projection.md @@ -261,7 +261,7 @@ interface FieldRef { Each source adapter exposes a field catalog for known fields. A catalog entry declares: -- logical name and scalar type; +- logical name and one or more scalar types; - allowed selector operators; - sensitivity and whether a projector may expose it by default; - whether historical replay is `exact`, `coerced`, or `unavailable`; @@ -272,7 +272,11 @@ reference an uncatalogued attribute by explicitly declaring its expected scalar type. At runtime a differently typed value does not get coerced; it does not match, and a bounded type-mismatch metric is recorded. Source-specific modules should publish catalogs for common attributes so users do not need to repeat -those declarations. +those declarations. OTLP log bodies are deliberately closed in version 1: only +the polymorphic `body:value` field is selectable, and only when the entire body +is a scalar. Structured body objects and arrays remain available to projectors +through normalized signal data but do not advertise child selector fields that +the adapter cannot populate. ### Selector AST @@ -285,12 +289,12 @@ type SignalPredicate = | { readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" readonly field: FieldRef - readonly value: SignalScalar + readonly value: SignalLiteral } | { readonly op: "in" readonly field: FieldRef - readonly values: readonly SignalScalar[] + readonly values: readonly SignalLiteral[] } ``` @@ -325,7 +329,12 @@ selector to: - no regular expressions, functions, arithmetic, joins, or user code. These bounds keep evaluation predictable and leave room for indexing active -projections by source kind and simple discriminating fields. +projections by source kind and simple discriminating fields. `SignalScalar` +describes normalized source data and does not inherit the literal-only 4 KiB +limit; the OTLP adapter accepts source strings up to its separate 16 KiB bound. +The literal limit is normative in UTF-8 bytes. Because JSON Schema `maxLength` +counts characters rather than encoded bytes, the generated schema documents the +constraint and the shared multibyte conformance vectors enforce its exact edge. ### Signal projection @@ -878,15 +887,17 @@ FULL`. While ingest is quiesced, backup first completes and verifies a blocking Maple Local activates immutable revisions with authenticated `POST /local/eventing/projections`. The same maintenance credential protects `GET /local/eventing/projections`, `/local/eventing/health`, and -`/local/eventing/outbox`. The outbox endpoint returns ready records with their -monotonic `sequence` by default; `?after=&limit=` pages beyond the -oldest page, while `?state=staged` exposes bounded inspection of records stranded -before the chDB commit point. The Local store defaults to at most 10,000 events -and 256 MiB of canonical event JSON. Staging fails closed with a retryable ingest -error before either cap can be exceeded. These endpoints are an operable -inspection/recovery surface, not yet a delivery protocol: durable consumer -claim/acknowledgement and deletion/retention begin with the first downstream -consumer rather than being implied by a destructive read API in this PR. +`/local/eventing/outbox`. Ready records receive a separate, append-only +readiness `sequence` on their first staged-to-ready transition; +`?after=&limit=` therefore cannot skip an older staged event that is +recovered after newer events were already read. `?state=staged` uses the original +staging sequence for bounded inspection of records stranded before the chDB +commit point. The Local store defaults to at most 10,000 events and 256 MiB of +canonical event JSON. Staging fails closed with a retryable ingest error before +either cap can be exceeded. These endpoints are an operable inspection/recovery +surface, not yet a delivery protocol: durable consumer claim/acknowledgement and +deletion/retention begin with the first downstream consumer rather than being +implied by a destructive read API in this PR. Re-delivery is the safe recovery operation: it deduplicates the same staged event ID and promotes it only after the warehouse write succeeds. Maple never blindly diff --git a/packages/eventing-core/fixtures/v1.json b/packages/eventing-core/fixtures/v1.json index dfc1c66da..9bd689b6d 100644 --- a/packages/eventing-core/fixtures/v1.json +++ b/packages/eventing-core/fixtures/v1.json @@ -14,6 +14,20 @@ "output": "sha256:061c0b5d99b92ef65ab8813c6d84988e4b1582e705e0077c952e62a0e84b6b08" } ], + "stringLiteralByteVectors": [ + { + "name": "multibyte literal exactly at the UTF-8 byte limit", + "unit": "é", + "repeat": 2048, + "valid": true + }, + { + "name": "multibyte literal one code point beyond the UTF-8 byte limit", + "unit": "é", + "repeat": 2049, + "valid": false + } + ], "predicateVectors": [ { "name": "int64 remains exact above JavaScript safe integer range", diff --git a/packages/eventing-core/schemas/signal-projection.v1.schema.json b/packages/eventing-core/schemas/signal-projection.v1.schema.json index f9d9b935f..0523586da 100644 --- a/packages/eventing-core/schemas/signal-projection.v1.schema.json +++ b/packages/eventing-core/schemas/signal-projection.v1.schema.json @@ -29,7 +29,7 @@ "required": ["namespace", "key", "type"], "additionalProperties": false }, - "SignalScalar": { + "SignalLiteral": { "anyOf": [ { "type": "object", @@ -40,11 +40,7 @@ }, "value": { "type": "string", - "allOf": [ - { - "maxLength": 4096 - } - ] + "description": "Predicate string literal limited to 4096 UTF-8 bytes; JSON Schema cannot express this byte-count constraint" } }, "required": ["type", "value"], @@ -234,7 +230,7 @@ "$ref": "#/$defs/SignalFieldRef" }, "value": { - "$ref": "#/$defs/SignalScalar" + "$ref": "#/$defs/SignalLiteral" } }, "required": ["op", "field", "value"], @@ -253,7 +249,7 @@ "values": { "type": "array", "items": { - "$ref": "#/$defs/SignalScalar" + "$ref": "#/$defs/SignalLiteral" }, "allOf": [ { diff --git a/packages/eventing-core/schemas/signal-scalar.v1.schema.json b/packages/eventing-core/schemas/signal-scalar.v1.schema.json index 81fa5bf22..d83d0d9a2 100644 --- a/packages/eventing-core/schemas/signal-scalar.v1.schema.json +++ b/packages/eventing-core/schemas/signal-scalar.v1.schema.json @@ -13,12 +13,7 @@ "enum": ["string"] }, "value": { - "type": "string", - "allOf": [ - { - "maxLength": 4096 - } - ] + "type": "string" } }, "required": ["type", "value"], diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts index 48fd45dbb..b298e8b77 100644 --- a/packages/eventing-core/src/model.ts +++ b/packages/eventing-core/src/model.ts @@ -6,6 +6,8 @@ export const MAX_IN_VALUES = 100 export const MAX_STRING_LITERAL_BYTES = 4 * 1024 export const MAX_DECIMAL_INT64_LENGTH = 20 +const utf8Bytes = (value: string): number => new TextEncoder().encode(value).byteLength + const NonEmptyIdentifier = Schema.String.check( Schema.isMinLength(1), Schema.isMaxLength(256), @@ -23,7 +25,21 @@ const Rfc3339Timestamp = Schema.String.check( export const StringSignalScalar = Schema.Struct({ type: Schema.Literal("string"), - value: Schema.String.check(Schema.isMaxLength(MAX_STRING_LITERAL_BYTES)), + value: Schema.String, +}) + +const StringLiteralValue = Schema.String.annotate({ + description: `Predicate string literal limited to ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes; JSON Schema cannot express this byte-count constraint`, +}).check( + Schema.makeFilter((value) => utf8Bytes(value) <= MAX_STRING_LITERAL_BYTES, { + expected: `a string no larger than ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`, + description: `Predicate string literal limited to ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`, + }), +) + +export const StringSignalLiteral = Schema.Struct({ + type: Schema.Literal("string"), + value: StringLiteralValue, }) export const BooleanSignalScalar = Schema.Struct({ @@ -62,6 +78,16 @@ export const SignalScalarSchema = Schema.Union([ export type SignalScalar = Schema.Schema.Type export type SignalScalarType = SignalScalar["type"] +export const SignalLiteralSchema = Schema.Union([ + StringSignalLiteral, + BooleanSignalScalar, + Int64SignalScalar, + Float64SignalScalar, + TimestampSignalScalar, + DurationSignalScalar, +]).annotate({ identifier: "SignalLiteral" }) +export type SignalLiteral = Schema.Schema.Type + export const FieldNamespaceSchema = Schema.Literals(["signal", "resource", "scope", "attribute", "body"]) export type FieldNamespace = Schema.Schema.Type @@ -95,13 +121,13 @@ export interface ExistsPredicate { export interface ComparisonPredicate { readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" readonly field: FieldRef - readonly value: SignalScalar + readonly value: SignalLiteral } export interface InPredicate { readonly op: "in" readonly field: FieldRef - readonly values: readonly SignalScalar[] + readonly values: readonly SignalLiteral[] } export type SignalPredicate = @@ -140,12 +166,12 @@ export const SignalPredicateSchema: Schema.Codec + readonly stringLiteralByteVectors: ReadonlyArray<{ + readonly name: string + readonly unit: string + readonly repeat: number + readonly valid: boolean + }> readonly predicateVectors: ReadonlyArray<{ readonly name: string readonly predicate: unknown @@ -70,6 +77,15 @@ describe("cross-language conformance vectors", () => { }) } + for (const vector of fixture.stringLiteralByteVectors) { + it(`string literal bytes: ${vector.name}`, () => { + const candidate = { type: "string", value: vector.unit.repeat(vector.repeat) } + if (vector.valid) + expect(() => Schema.decodeUnknownSync(SignalLiteralSchema)(candidate)).not.toThrow() + else expect(() => Schema.decodeUnknownSync(SignalLiteralSchema)(candidate)).toThrow() + }) + } + for (const vector of fixture.predicateVectors) { it(`predicate: ${vector.name}`, () => { const predicate = Schema.decodeUnknownSync(SignalPredicateSchema)(vector.predicate) @@ -158,6 +174,31 @@ describe("selector validation", () => { }) describe("total runtime behavior", () => { + it("accepts valid large source strings for exists and small contains literals", () => { + const largeValue = `${"a".repeat(5 * 1024)}needle` + const signal = signalFor([ + { + namespace: "attribute", + key: "large.description", + value: { type: "string", value: largeValue }, + }, + ]) + const field = { + namespace: "attribute" as const, + key: "large.description", + type: "string" as const, + } + + expect(compileSignalPredicate({ op: "exists", field })(signal).matches).toBe(true) + expect( + compileSignalPredicate({ + op: "contains", + field, + value: { type: "string", value: "needle" }, + })(signal).matches, + ).toBe(true) + }) + it("treats malformed source scalars as mismatches rather than throwing", () => { const field: FieldRef = { namespace: "attribute", key: "n", type: "int64" } const evaluate = compileSignalPredicate({ diff --git a/packages/eventing-core/src/predicate.ts b/packages/eventing-core/src/predicate.ts index 850c0ca19..d63a7b201 100644 --- a/packages/eventing-core/src/predicate.ts +++ b/packages/eventing-core/src/predicate.ts @@ -1,6 +1,7 @@ import type { FieldRef, NormalizedSignal, + SignalLiteral, SignalPredicate, SignalProjectionSpec, SignalScalar, @@ -173,9 +174,6 @@ export const validateSignalScalar = (scalar: SignalScalar, path = "value"): read const issues: ValidationIssue[] = [] switch (scalar.type) { case "string": - if (stringBytes(scalar.value) > MAX_STRING_LITERAL_BYTES) - issues.push({ path, message: `string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes` }) - break case "boolean": break case "int64": @@ -197,6 +195,13 @@ export const validateSignalScalar = (scalar: SignalScalar, path = "value"): read return issues } +export const validateSignalLiteral = (literal: SignalLiteral, path = "value"): readonly ValidationIssue[] => [ + ...validateSignalScalar(literal, path), + ...(literal.type === "string" && stringBytes(literal.value) > MAX_STRING_LITERAL_BYTES + ? [{ path, message: `string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes` }] + : []), +] + export const validateSignalPredicate = (predicate: SignalPredicate): readonly ValidationIssue[] => { const issues: ValidationIssue[] = [] let nodes = 0 @@ -228,7 +233,7 @@ export const validateSignalPredicate = (predicate: SignalPredicate): readonly Va case "contains": if (node.field.type !== "string" || node.value.type !== "string") issues.push({ path, message: "contains requires a string field and string literal" }) - issues.push(...validateSignalScalar(node.value, `${path}.value`)) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) break case "gt": case "gte": @@ -238,13 +243,13 @@ export const validateSignalPredicate = (predicate: SignalPredicate): readonly Va issues.push({ path, message: `${node.op} is not supported for ${node.field.type}` }) if (node.field.type !== node.value.type) issues.push({ path, message: "field and literal types must match" }) - issues.push(...validateSignalScalar(node.value, `${path}.value`)) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) break case "eq": case "neq": if (node.field.type !== node.value.type) issues.push({ path, message: "field and literal types must match" }) - issues.push(...validateSignalScalar(node.value, `${path}.value`)) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) break case "in": if (node.values.length === 0) @@ -258,7 +263,7 @@ export const validateSignalPredicate = (predicate: SignalPredicate): readonly Va path: `${path}.values[${i}]`, message: "field and literal types must match", }) - issues.push(...validateSignalScalar(value, `${path}.values[${i}]`)) + issues.push(...validateSignalLiteral(value, `${path}.values[${i}]`)) } break } diff --git a/packages/eventing-core/src/source.ts b/packages/eventing-core/src/source.ts index a9a6a7ee2..cc1a722f7 100644 --- a/packages/eventing-core/src/source.ts +++ b/packages/eventing-core/src/source.ts @@ -5,13 +5,21 @@ import type { ValidationIssue } from "./predicate" export type SignalLeafOperator = "exists" | "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" | "in" export type ReplayCapability = "exact" | "coerced" | "unavailable" -export interface SignalFieldCatalogEntry { - readonly field: FieldRef +interface SignalFieldCatalogEntryBase { readonly operators: readonly SignalLeafOperator[] readonly sensitivity: "public" | "sensitive" readonly replay: ReplayCapability } +export type SignalFieldCatalogEntry = SignalFieldCatalogEntryBase & + ( + | { readonly field: FieldRef; readonly types?: never } + | { + readonly field: Pick + readonly types: readonly SignalScalarType[] + } + ) + export interface OpenFieldNamespacePolicy { readonly namespace: FieldNamespace readonly types: readonly SignalScalarType[] @@ -37,6 +45,11 @@ interface RegisteredSignalSource { readonly openFields: ReadonlyMap } +const catalogEntryTypes = (entry: SignalFieldCatalogEntry): readonly SignalScalarType[] => { + if (entry.types !== undefined) return entry.types + return [entry.field.type] +} + export class SignalSourceRegistry { readonly #sources = new Map() @@ -51,6 +64,8 @@ export class SignalSourceRegistry { if (fields.has(key)) throw new Error(`duplicate field catalog entry: ${definition.sourceKind}:${key}`) if (entry.operators.length === 0) throw new Error(`field catalog entry has no operators: ${key}`) + if (entry.types !== undefined && entry.types.length === 0) + throw new Error(`field catalog entry has no types: ${key}`) fields.set(key, entry) } @@ -105,10 +120,11 @@ export const validatePredicateAgainstSource = ( for (const leaf of leafFields(predicate)) { const catalogEntry = source.fields.get(fieldKey(leaf.field)) if (catalogEntry) { - if (catalogEntry.field.type !== leaf.field.type) + const catalogTypes = catalogEntryTypes(catalogEntry) + if (!catalogTypes.includes(leaf.field.type)) issues.push({ path: `${leaf.path}.field.type`, - message: `catalog field ${fieldKey(leaf.field)} has type ${catalogEntry.field.type}`, + message: `catalog field ${fieldKey(leaf.field)} allows ${catalogTypes.join(", ")}`, }) if (!catalogEntry.operators.includes(leaf.operator)) issues.push({ From 678f115efe1ecc6d45a5920351a260bed638c6d8 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 13 Aug 2026 19:33:32 -0400 Subject: [PATCH 05/12] refactor(cli): isolate GitLab event projectors --- .../src/server/eventing/gitlab-projectors.ts | 107 ++++++++++++++++++ apps/cli/src/server/eventing/runtime.ts | 104 +---------------- 2 files changed, 109 insertions(+), 102 deletions(-) create mode 100644 apps/cli/src/server/eventing/gitlab-projectors.ts diff --git a/apps/cli/src/server/eventing/gitlab-projectors.ts b/apps/cli/src/server/eventing/gitlab-projectors.ts new file mode 100644 index 000000000..d50518bf2 --- /dev/null +++ b/apps/cli/src/server/eventing/gitlab-projectors.ts @@ -0,0 +1,107 @@ +import { + ProjectorRegistry, + fieldKey, + isJsonValue, + type NormalizedSignal, + type SignalScalar, +} from "@maple/eventing-core" + +export interface GitLabIssueProjectorConfig { + readonly includeBody: boolean +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const gitlabProjectorConfig = (value: unknown): GitLabIssueProjectorConfig => { + if (!isRecord(value)) throw new Error("gitlab.issue.created projector config must be an object") + const keys = Object.keys(value) + if (keys.some((key) => key !== "includeBody")) + throw new Error("gitlab.issue.created projector config contains an unknown field") + if (value.includeBody !== undefined && typeof value.includeBody !== "boolean") + throw new Error("gitlab.issue.created includeBody must be boolean") + return { includeBody: value.includeBody === true } +} + +const field = (signal: NormalizedSignal, namespace: "resource" | "attribute", key: string) => + signal.fields.get(fieldKey({ namespace, key })) + +const scalarString = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "string") throw new Error(`GitLab issue event ${label} must be a string`) + return value.value +} + +const scalarInt64 = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "int64") throw new Error(`GitLab issue event ${label} must be an int64`) + return value.value +} + +const gitlabIssueProjector = { + id: "gitlab.issue.created", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.created.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-created:v1", + decodeConfig: gitlabProjectorConfig, + project: (signal: NormalizedSignal, config: GitLabIssueProjectorConfig) => { + const projectId = scalarInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") + const projectPath = scalarString( + field(signal, "attribute", "gitlab.project.path"), + "gitlab.project.path", + true, + )! + const issueId = scalarInt64(field(signal, "attribute", "gitlab.issue.id"), "gitlab.issue.id") + const issueIid = scalarInt64( + field(signal, "attribute", "gitlab.issue.iid"), + "gitlab.issue.iid", + true, + )! + const title = scalarString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") + const url = scalarString(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") + const actorId = scalarInt64(field(signal, "attribute", "gitlab.user.id"), "gitlab.user.id") + const actorUsername = scalarString( + field(signal, "attribute", "gitlab.user.username"), + "gitlab.user.username", + ) + const serviceName = scalarString(field(signal, "resource", "service.name"), "service.name") + const candidateBody = + isRecord(signal.data) && isRecord(signal.data.record) ? signal.data.record.body : undefined + const body = isJsonValue(candidateBody) ? candidateBody : undefined + return { + subject: `${projectPath}/issues/${issueIid}`, + data: { + project: { + ...(projectId === undefined ? {} : { id: projectId }), + path: projectPath, + }, + issue: { + ...(issueId === undefined ? {} : { id: issueId }), + iid: issueIid, + ...(title === undefined ? {} : { title }), + ...(url === undefined ? {} : { url }), + }, + ...(actorId === undefined && actorUsername === undefined + ? {} + : { + actor: { + ...(actorId === undefined ? {} : { id: actorId }), + ...(actorUsername === undefined ? {} : { username: actorUsername }), + }, + }), + ...(serviceName === undefined ? {} : { serviceName }), + ...(config.includeBody && body !== undefined ? { body } : {}), + }, + } + }, +} as const + +export const registerGitLabProjectors = (registry: ProjectorRegistry): ProjectorRegistry => + registry.register(gitlabIssueProjector) diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts index fcefd51b9..2b7085392 100644 --- a/apps/cli/src/server/eventing/runtime.ts +++ b/apps/cli/src/server/eventing/runtime.ts @@ -3,118 +3,18 @@ import { ProjectorRegistry, SignalSourceRegistry, assertSignalProjectionInputBudget, - fieldKey, - isJsonValue, SignalProjectionSpecSchema, type MapleCloudEvent, - type NormalizedSignal, type ProjectionFailure, type SignalProjectionSpec, - type SignalScalar, } from "@maple/eventing-core" import { Schema } from "effect" import { LocalEventingControlStore } from "./control-store" +import { registerGitLabProjectors } from "./gitlab-projectors" import { OTLP_LOG_ADAPTER } from "./otlp" const TENANT_ID = "local" -interface GitLabIssueProjectorConfig { - readonly includeBody: boolean -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const gitlabProjectorConfig = (value: unknown): GitLabIssueProjectorConfig => { - if (!isRecord(value)) throw new Error("gitlab.issue.created projector config must be an object") - const keys = Object.keys(value) - if (keys.some((key) => key !== "includeBody")) - throw new Error("gitlab.issue.created projector config contains an unknown field") - if (value.includeBody !== undefined && typeof value.includeBody !== "boolean") - throw new Error("gitlab.issue.created includeBody must be boolean") - return { includeBody: value.includeBody === true } -} - -const field = (signal: NormalizedSignal, namespace: "resource" | "attribute", key: string) => - signal.fields.get(fieldKey({ namespace, key })) - -const scalarString = (value: SignalScalar | undefined, label: string, required = false) => { - if (value === undefined) { - if (required) throw new Error(`GitLab issue event is missing ${label}`) - return undefined - } - if (value.type !== "string") throw new Error(`GitLab issue event ${label} must be a string`) - return value.value -} - -const scalarInt64 = (value: SignalScalar | undefined, label: string, required = false) => { - if (value === undefined) { - if (required) throw new Error(`GitLab issue event is missing ${label}`) - return undefined - } - if (value.type !== "int64") throw new Error(`GitLab issue event ${label} must be an int64`) - return value.value -} - -const gitlabIssueProjector = { - id: "gitlab.issue.created", - version: 1, - sourceKinds: ["otel.log"], - outputType: "dev.maple.gitlab.issue.created.v1", - dataSchema: "urn:maple:event-schema:gitlab-issue-created:v1", - decodeConfig: gitlabProjectorConfig, - project: (signal: NormalizedSignal, config: GitLabIssueProjectorConfig) => { - const projectId = scalarInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") - const projectPath = scalarString( - field(signal, "attribute", "gitlab.project.path"), - "gitlab.project.path", - true, - )! - const issueId = scalarInt64(field(signal, "attribute", "gitlab.issue.id"), "gitlab.issue.id") - const issueIid = scalarInt64( - field(signal, "attribute", "gitlab.issue.iid"), - "gitlab.issue.iid", - true, - )! - const title = scalarString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") - const url = scalarString(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") - const actorId = scalarInt64(field(signal, "attribute", "gitlab.user.id"), "gitlab.user.id") - const actorUsername = scalarString( - field(signal, "attribute", "gitlab.user.username"), - "gitlab.user.username", - ) - const serviceName = scalarString(field(signal, "resource", "service.name"), "service.name") - const candidateBody = - isRecord(signal.data) && isRecord(signal.data.record) ? signal.data.record.body : undefined - const body = isJsonValue(candidateBody) ? candidateBody : undefined - return { - subject: `${projectPath}/issues/${issueIid}`, - data: { - project: { - ...(projectId === undefined ? {} : { id: projectId }), - path: projectPath, - }, - issue: { - ...(issueId === undefined ? {} : { id: issueId }), - iid: issueIid, - ...(title === undefined ? {} : { title }), - ...(url === undefined ? {} : { url }), - }, - ...(actorId === undefined && actorUsername === undefined - ? {} - : { - actor: { - ...(actorId === undefined ? {} : { id: actorId }), - ...(actorUsername === undefined ? {} : { username: actorUsername }), - }, - }), - ...(serviceName === undefined ? {} : { serviceName }), - ...(config.includeBody && body !== undefined ? { body } : {}), - }, - } - }, -} as const - export interface LocalProjectionEvaluation { readonly events: readonly MapleCloudEvent[] readonly failures: readonly ProjectionFailure[] @@ -145,7 +45,7 @@ export class LocalEventingRuntime { constructor(store: LocalEventingControlStore) { this.#store = store this.#sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) - this.#projectors = new ProjectorRegistry().register(gitlabIssueProjector) + this.#projectors = registerGitLabProjectors(new ProjectorRegistry()) const specs = store.loadEnabledProjections(TENANT_ID) this.#compiled = CompiledProjectionRegistry.compile(specs, this.#sources, this.#projectors) this.#activeSourceKinds = new Set(specs.map(({ sourceKind }) => sourceKind)) From f3563e1c0b9d26834a67fd65dcde70358945d3fb Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 13 Aug 2026 19:42:14 -0400 Subject: [PATCH 06/12] feat(cli): project GitLab lifecycle events --- .../src/server/eventing/gitlab-projectors.ts | 306 ++++++++++++++++- .../test/fixtures/gitlab-projectors.v1.json | 89 +++++ apps/cli/test/gitlab-projectors.test.ts | 315 ++++++++++++++++++ docs/gitlab-event-projectors.md | 168 ++++++++++ 4 files changed, 875 insertions(+), 3 deletions(-) create mode 100644 apps/cli/test/fixtures/gitlab-projectors.v1.json create mode 100644 apps/cli/test/gitlab-projectors.test.ts create mode 100644 docs/gitlab-event-projectors.md diff --git a/apps/cli/src/server/eventing/gitlab-projectors.ts b/apps/cli/src/server/eventing/gitlab-projectors.ts index d50518bf2..79af999da 100644 --- a/apps/cli/src/server/eventing/gitlab-projectors.ts +++ b/apps/cli/src/server/eventing/gitlab-projectors.ts @@ -23,7 +23,7 @@ const gitlabProjectorConfig = (value: unknown): GitLabIssueProjectorConfig => { return { includeBody: value.includeBody === true } } -const field = (signal: NormalizedSignal, namespace: "resource" | "attribute", key: string) => +const field = (signal: NormalizedSignal, namespace: "signal" | "resource" | "attribute", key: string) => signal.fields.get(fieldKey({ namespace, key })) const scalarString = (value: SignalScalar | undefined, label: string, required = false) => { @@ -44,7 +44,7 @@ const scalarInt64 = (value: SignalScalar | undefined, label: string, required = return value.value } -const gitlabIssueProjector = { +export const gitlabIssueCreatedProjector = { id: "gitlab.issue.created", version: 1, sourceKinds: ["otel.log"], @@ -103,5 +103,305 @@ const gitlabIssueProjector = { }, } as const +type GitLabProjectAction = + | "created" + | "destroyed" + | "renamed" + | "transferred" + | "updated" + | "archived" + | "unarchived" + | "deletion_requested" + +const PROJECT_LIFECYCLE_ACTIONS: Readonly> = { + project_create: "created", + project_destroy: "destroyed", + project_rename: "renamed", + project_transfer: "transferred", + project_update: "updated", + project_archive: "archived", + project_unarchive: "unarchived", + project_deletion_request: "deletion_requested", +} + +const TERMINAL_PIPELINE_STATUSES = new Set(["success", "failed", "canceled", "skipped"]) +const MAX_PROJECTOR_TEXT_BYTES = 4 * 1024 + +const projectorString = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + if (value === undefined) { + if (required) throw new Error(`GitLab projector is missing ${label}`) + return undefined + } + if (value.type !== "string") throw new Error(`GitLab projector ${label} must be a string`) + const text = value.value.trim() + if (text.length === 0) throw new Error(`GitLab projector ${label} must not be blank`) + if (Buffer.byteLength(text, "utf8") > MAX_PROJECTOR_TEXT_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_PROJECTOR_TEXT_BYTES} UTF-8 bytes`) + return text +} + +const projectorInt64 = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + if (value === undefined) { + if (required) throw new Error(`GitLab projector is missing ${label}`) + return undefined + } + if (value.type !== "int64") throw new Error(`GitLab projector ${label} must be an int64`) + return value.value +} + +const positiveProjectorInt64 = (value: SignalScalar | undefined, label: string): string => { + const parsed = projectorInt64(value, label, true)! + if (BigInt(parsed) <= 0n) throw new Error(`GitLab projector ${label} must be positive`) + return parsed +} + +const optionalPositiveProjectorInt64 = ( + value: SignalScalar | undefined, + label: string, +): string | undefined => { + const parsed = projectorInt64(value, label) + if (parsed !== undefined && BigInt(parsed) <= 0n) + throw new Error(`GitLab projector ${label} must be positive`) + return parsed +} + +const nonNegativeProjectorInt64 = (value: SignalScalar | undefined, label: string): string | undefined => { + const parsed = projectorInt64(value, label) + if (parsed !== undefined && BigInt(parsed) < 0n) + throw new Error(`GitLab projector ${label} must not be negative`) + return parsed +} + +const eventName = (signal: NormalizedSignal): string => + projectorString( + field(signal, "attribute", "event.name") ?? field(signal, "signal", "event.name"), + "event.name", + true, + )! + +const actor = (signal: NormalizedSignal) => { + const id = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.actor.id"), + "gitlab.actor.id", + ) + const name = projectorString(field(signal, "attribute", "gitlab.actor.name"), "gitlab.actor.name") + if (name === undefined && id === undefined) return undefined + return { + ...(id === undefined ? {} : { id }), + ...(name === undefined ? {} : { name }), + } +} + +const project = (signal: NormalizedSignal) => { + const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") + const path = projectorString( + field(signal, "attribute", "gitlab.project.path"), + "gitlab.project.path", + true, + )! + const oldPath = projectorString( + field(signal, "attribute", "gitlab.project.old_path"), + "gitlab.project.old_path", + ) + return { + id, + path, + ...(oldPath === undefined ? {} : { oldPath }), + } +} + +const serviceName = (signal: NormalizedSignal): string | undefined => + projectorString(field(signal, "resource", "service.name"), "service.name") + +const result = (signal: NormalizedSignal): string | undefined => + projectorString(field(signal, "attribute", "gitlab.event.result"), "gitlab.event.result") + +const noProjectorConfig = + (projectorId: string) => + (value: unknown): Record => { + if (!isRecord(value)) throw new Error(`${projectorId} projector config must be an object`) + if (Object.keys(value).length > 0) + throw new Error(`${projectorId} projector config contains unknown fields`) + return {} + } + +export const gitlabProjectLifecycleProjector = { + id: "gitlab.project.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.project.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-project-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.project.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const action = PROJECT_LIFECYCLE_ACTIONS[sourceEvent] + if (action === undefined) + throw new Error(`GitLab projector does not recognize project event ${sourceEvent}`) + const projectData = project(signal) + const eventActor = actor(signal) + const eventServiceName = serviceName(signal) + const eventResult = result(signal) + return { + subject: projectData.path, + data: { + project: projectData, + action, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabMergeRequestLifecycleProjector = { + id: "gitlab.merge-request.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.merge-request.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.merge-request.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const match = /^merge_request_([a-z][a-z0-9_]{0,63})$/.exec(sourceEvent) + if (!match) throw new Error(`GitLab projector does not recognize merge request event ${sourceEvent}`) + const projectData = project(signal) + const iid = positiveProjectorInt64( + field(signal, "attribute", "gitlab.merge_request.iid"), + "gitlab.merge_request.iid", + ) + const sourceBranch = projectorString( + field(signal, "attribute", "gitlab.merge_request.source_branch"), + "gitlab.merge_request.source_branch", + ) + const targetBranch = projectorString( + field(signal, "attribute", "gitlab.merge_request.target_branch"), + "gitlab.merge_request.target_branch", + ) + const commit = projectorString( + field(signal, "attribute", "gitlab.merge_request.commit"), + "gitlab.merge_request.commit", + ) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/merge_requests/${iid}`, + data: { + project: projectData, + mergeRequest: { + iid, + action: match[1]!, + ...(sourceBranch === undefined ? {} : { sourceBranch }), + ...(targetBranch === undefined ? {} : { targetBranch }), + ...(commit === undefined ? {} : { commit }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabPipelineCompletedProjector = { + id: "gitlab.pipeline.completed", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.pipeline.completed.v1", + dataSchema: "urn:maple:event-schema:gitlab-pipeline-completed:v1", + decodeConfig: noProjectorConfig("gitlab.pipeline.completed"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + if (sourceEvent !== "ci_pipeline_completed") + throw new Error(`GitLab projector does not recognize pipeline event ${sourceEvent}`) + const projectData = project(signal) + const id = positiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.id"), + "gitlab.ci.pipeline.id", + ) + const iid = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.iid"), + "gitlab.ci.pipeline.iid", + ) + const status = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.status"), + "gitlab.ci.pipeline.status", + true, + )! + if (!TERMINAL_PIPELINE_STATUSES.has(status)) + throw new Error(`GitLab projector pipeline status ${status} is not terminal`) + const name = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.name"), + "gitlab.ci.pipeline.name", + ) + const pipelineSource = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.source"), + "gitlab.ci.pipeline.source", + ) + const detailedStatus = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.detailed_status"), + "gitlab.ci.pipeline.detailed_status", + ) + const ref = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref") + const sha = projectorString( + field(signal, "attribute", "vcs.ref.head.revision"), + "vcs.ref.head.revision", + ) + const durationMs = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.duration_ms"), + "gitlab.ci.pipeline.duration_ms", + ) + const queuedDurationMs = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.queued_duration_ms"), + "gitlab.ci.pipeline.queued_duration_ms", + ) + const stageCount = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.stage_count"), + "gitlab.ci.pipeline.stage_count", + ) + const eventActor = actor(signal) + const eventServiceName = serviceName(signal) + const eventResult = result(signal) + return { + subject: `${projectData.path}/-/pipelines/${id}`, + data: { + project: projectData, + pipeline: { + id, + ...(iid === undefined ? {} : { iid }), + status, + ...(name === undefined ? {} : { name }), + ...(pipelineSource === undefined ? {} : { source: pipelineSource }), + ...(detailedStatus === undefined ? {} : { detailedStatus }), + ...(ref === undefined ? {} : { ref }), + ...(sha === undefined ? {} : { sha }), + ...(durationMs === undefined ? {} : { durationMs }), + ...(queuedDurationMs === undefined ? {} : { queuedDurationMs }), + ...(stageCount === undefined ? {} : { stageCount }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + export const registerGitLabProjectors = (registry: ProjectorRegistry): ProjectorRegistry => - registry.register(gitlabIssueProjector) + registry + .register(gitlabIssueCreatedProjector) + .register(gitlabProjectLifecycleProjector) + .register(gitlabMergeRequestLifecycleProjector) + .register(gitlabPipelineCompletedProjector) diff --git a/apps/cli/test/fixtures/gitlab-projectors.v1.json b/apps/cli/test/fixtures/gitlab-projectors.v1.json new file mode 100644 index 000000000..003a013ee --- /dev/null +++ b/apps/cli/test/fixtures/gitlab-projectors.v1.json @@ -0,0 +1,89 @@ +[ + { + "specversion": "1.0", + "id": "sha256:0fac0a375c6f8a05fb5ec1a751cf9e3b55282f56c0d135bc8a8c50c6a2a7559f", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.project.lifecycle.v1", + "subject": "rdev/maple", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-project-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-project-renamed", + "projectionrevision": 1, + "projectorid": "gitlab.project.lifecycle", + "projectorversion": 1, + "data": { + "project": { "id": "42", "path": "rdev/maple", "oldPath": "rdev/old-maple" }, + "action": "renamed", + "sourceEvent": "project_rename", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:09992cc6804e056fb2a037b6c9db05c432c4733e229c963675a15a47445f745b", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.merge-request.lifecycle.v1", + "subject": "rdev/maple/-/merge_requests/7", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-mr-opened", + "projectionrevision": 1, + "projectorid": "gitlab.merge-request.lifecycle", + "projectorversion": 1, + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "mergeRequest": { + "iid": "7", + "action": "open", + "sourceBranch": "feature/maple", + "targetBranch": "main", + "commit": "abc123" + }, + "sourceEvent": "merge_request_open", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:f2bab7e420aa6b97d65f0478d16384b45d3bf61fb9ad9c111c614279cb42e24e", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.pipeline.completed.v1", + "subject": "rdev/maple/-/pipelines/900", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-pipeline-completed:v1", + "tenantid": "local", + "projectionid": "gitlab-pipeline-completed", + "projectionrevision": 1, + "projectorid": "gitlab.pipeline.completed", + "projectorversion": 1, + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "pipeline": { + "id": "900", + "iid": "12", + "status": "failed", + "name": "Maple CI", + "source": "push", + "detailedStatus": "failed", + "ref": "main", + "sha": "deadbeef", + "durationMs": "63000", + "queuedDurationMs": "10000", + "stageCount": "3" + }, + "sourceEvent": "ci_pipeline_completed", + "actor": { "id": "9", "name": "rdev" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + } +] diff --git a/apps/cli/test/gitlab-projectors.test.ts b/apps/cli/test/gitlab-projectors.test.ts new file mode 100644 index 000000000..6707b8eaf --- /dev/null +++ b/apps/cli/test/gitlab-projectors.test.ts @@ -0,0 +1,315 @@ +import { deepStrictEqual, strictEqual, throws } from "node:assert" +import { describe, it } from "vitest" +import { + CompiledProjectionRegistry, + ProjectorRegistry, + SignalSourceRegistry, + validateMapleCloudEvent, + type SignalPredicate, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import samples from "./fixtures/gitlab-projectors.v1.json" +import { + gitlabMergeRequestLifecycleProjector, + gitlabPipelineCompletedProjector, + gitlabProjectLifecycleProjector, + registerGitLabProjectors, +} from "../src/server/eventing/gitlab-projectors" +import { OTLP_LOG_ADAPTER, normalizeOtlpLogs } from "../src/server/eventing/otlp" + +const stringAttr = (key: string, value: string) => ({ key, value: { stringValue: value } }) +const intAttr = (key: string, value: string) => ({ key, value: { intValue: value } }) + +const gitlabEvent = (eventName: string, eventId: string, extra: readonly unknown[] = []) => ({ + resourceLogs: [ + { + resource: { + attributes: [ + stringAttr("service.name", "gitlab-repository-events"), + stringAttr("service.version", "19.1.0"), + ], + }, + scopeLogs: [ + { + scope: { name: "srvmini2.gitlab.repository-events", version: "1" }, + logRecords: [ + { + timeUnixNano: "1786131720123456789", + observedTimeUnixNano: "1786131721123456789", + severityNumber: 9, + severityText: "INFO", + body: { stringValue: `gitlab event ${eventName}` }, + attributes: [ + stringAttr("event.id", eventId), + stringAttr("event.source", "https://gitlab.internal"), + stringAttr("event.name", eventName), + stringAttr("gitlab.event.id", eventId), + stringAttr("gitlab.event.result", "success"), + intAttr("gitlab.project.id", "42"), + stringAttr("gitlab.project.path", "rdev/maple"), + intAttr("gitlab.actor.id", "9"), + stringAttr("gitlab.actor.name", "rdev"), + ...extra, + ], + }, + ], + }, + ], + }, + ], +}) + +const normalizedEvent = (request: unknown) => { + const [signal] = normalizeOtlpLogs(request, "2026-08-07T20:00:00Z") + if (!signal) throw new Error("test event did not normalize") + return signal +} + +const withoutAttribute = (request: ReturnType, key: string) => { + const copy = structuredClone(request) + copy.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.attributes = + copy.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.attributes.filter((entry) => entry.key !== key) + return copy +} + +const selectorFor = (eventName: string): SignalPredicate => ({ + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: eventName }, +}) + +const evaluate = ( + projectorId: string, + projectionId: string, + eventName: string, + signal: ReturnType, +) => { + const spec: SignalProjectionSpec = { + id: projectionId, + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: selectorFor(eventName), + projector: { id: projectorId, version: 1, config: {} }, + activeFrom: "2000-01-01T00:00:00Z", + } + const sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) + const projectors = registerGitLabProjectors(new ProjectorRegistry()) + return CompiledProjectionRegistry.compile([spec], sources, projectors).evaluate(signal) +} + +describe("GitLab event projectors", () => { + it("projects the production receiver's project lifecycle fields", () => { + const signal = normalizedEvent( + gitlabEvent("project_rename", "project-event-42", [ + stringAttr("gitlab.project.old_path", "rdev/old-maple"), + ]), + ) + const result = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-renamed", + "project_rename", + signal, + ) + strictEqual(result.failures.length, 0) + strictEqual(result.events[0]?.id, samples[0]?.id) + deepStrictEqual(result.events[0]?.data, { + project: { id: "42", path: "rdev/maple", oldPath: "rdev/old-maple" }, + action: "renamed", + sourceEvent: "project_rename", + actor: { id: "9", name: "rdev" }, + result: "success", + serviceName: "gitlab-repository-events", + }) + strictEqual(result.events[0]?.subject, "rdev/maple") + }) + + it("maps every normalized project lifecycle event to a version-1 action", () => { + const actions = { + project_create: "created", + project_destroy: "destroyed", + project_rename: "renamed", + project_transfer: "transferred", + project_update: "updated", + project_archive: "archived", + project_unarchive: "unarchived", + project_deletion_request: "deletion_requested", + } as const + for (const [sourceEvent, action] of Object.entries(actions)) { + const result = evaluate( + "gitlab.project.lifecycle", + `gitlab-project-${sourceEvent}`, + sourceEvent, + normalizedEvent(gitlabEvent(sourceEvent, `project-${sourceEvent}`)), + ) + strictEqual(result.failures.length, 0) + const event = result.events[0] + if (!event) throw new Error(`projector did not emit ${sourceEvent}`) + strictEqual((event.data as { readonly action?: string }).action, action) + } + }) + + it("projects merge-request lifecycle data without inventing absent fields", () => { + const signal = normalizedEvent( + gitlabEvent("merge_request_open", "mr-event-7", [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.source_branch", "feature/maple"), + stringAttr("gitlab.merge_request.target_branch", "main"), + stringAttr("gitlab.merge_request.commit", "abc123"), + ]), + ) + const result = evaluate( + "gitlab.merge-request.lifecycle", + "gitlab-mr-opened", + "merge_request_open", + signal, + ) + strictEqual(result.failures.length, 0) + strictEqual(result.events[0]?.id, samples[1]?.id) + deepStrictEqual(result.events[0]?.data, { + project: { id: "42", path: "rdev/maple" }, + mergeRequest: { + iid: "7", + action: "open", + sourceBranch: "feature/maple", + targetBranch: "main", + commit: "abc123", + }, + sourceEvent: "merge_request_open", + actor: { id: "9", name: "rdev" }, + result: "success", + serviceName: "gitlab-repository-events", + }) + strictEqual(result.events[0]?.subject, "rdev/maple/-/merge_requests/7") + }) + + it("projects only terminal pipeline events from the normalized receiver fields", () => { + const signal = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-event-900", [ + stringAttr("gitlab.event.result", "failure"), + intAttr("gitlab.ci.pipeline.id", "900"), + intAttr("gitlab.ci.pipeline.iid", "12"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + stringAttr("gitlab.ci.pipeline.detailed_status", "failed"), + stringAttr("gitlab.ci.pipeline.name", "Maple CI"), + stringAttr("gitlab.ci.pipeline.source", "push"), + stringAttr("vcs.ref", "main"), + stringAttr("vcs.ref.head.revision", "deadbeef"), + intAttr("gitlab.ci.pipeline.duration_ms", "63000"), + intAttr("gitlab.ci.pipeline.queued_duration_ms", "10000"), + intAttr("gitlab.ci.pipeline.stage_count", "3"), + ]), + ) + const result = evaluate( + "gitlab.pipeline.completed", + "gitlab-pipeline-completed", + "ci_pipeline_completed", + signal, + ) + strictEqual(result.failures.length, 0) + strictEqual(result.events[0]?.id, samples[2]?.id) + deepStrictEqual(result.events[0]?.data, { + project: { id: "42", path: "rdev/maple" }, + pipeline: { + id: "900", + iid: "12", + status: "failed", + name: "Maple CI", + source: "push", + detailedStatus: "failed", + ref: "main", + sha: "deadbeef", + durationMs: "63000", + queuedDurationMs: "10000", + stageCount: "3", + }, + sourceEvent: "ci_pipeline_completed", + actor: { id: "9", name: "rdev" }, + result: "failure", + serviceName: "gitlab-repository-events", + }) + strictEqual(result.events[0]?.subject, "rdev/maple/-/pipelines/900") + }) + + it("keeps source identity and the CloudEvent ID deterministic across retries", () => { + const first = normalizedEvent(gitlabEvent("project_create", "project-event-42")) + const retry = normalizedEvent(gitlabEvent("project_create", "project-event-42")) + const firstResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-created", + "project_create", + first, + ) + const retryResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-created", + "project_create", + retry, + ) + strictEqual(first.occurrenceId, "project-event-42") + strictEqual(retry.occurrenceId, "project-event-42") + strictEqual(firstResult.events[0]?.id, retryResult.events[0]?.id) + strictEqual( + firstResult.events[0]?.id, + "sha256:8f45527a4e615e057142026df477f20e2e6d63ae55bc761ce4dc151dda706811", + ) + strictEqual(firstResult.events[0]?.id?.startsWith("sha256:"), true) + }) + + it("fails closed on missing or malformed required fields", () => { + const missingProjectId = normalizedEvent( + withoutAttribute(gitlabEvent("project_create", "project-event-missing"), "gitlab.project.id"), + ) + const badProjectId = normalizedEvent( + gitlabEvent("project_create", "project-event-bad", [ + stringAttr("gitlab.project.id", "not-an-int"), + ]), + ) + const missingIdResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-missing", + "project_create", + missingProjectId, + ) + const badIdResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-bad", + "project_create", + badProjectId, + ) + strictEqual(missingIdResult.failures.length, 1) + strictEqual(missingIdResult.failures[0]?.message, "GitLab projector is missing gitlab.project.id") + strictEqual(badIdResult.failures.length, 1) + strictEqual(badIdResult.failures[0]?.message, "GitLab projector gitlab.project.id must be an int64") + }) + + it("rejects nonterminal pipeline status, unknown actions, and unknown config fields", () => { + const running = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-running", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "running"), + ]), + ) + throws( + () => gitlabPipelineCompletedProjector.project(running), + /GitLab projector pipeline status running is not terminal/, + ) + const unknownAction = normalizedEvent( + gitlabEvent("merge_request_??", "mr-invalid", [intAttr("gitlab.merge_request.iid", "7")]), + ) + throws( + () => gitlabMergeRequestLifecycleProjector.project(unknownAction), + /GitLab projector does not recognize merge request event/, + ) + throws( + () => gitlabProjectLifecycleProjector.decodeConfig({ includeBody: true }), + /gitlab\.project\.lifecycle projector config contains unknown fields/, + ) + }) + + it("keeps the checked-in sample CloudEvents envelope-valid", () => { + strictEqual(samples.length, 3) + for (const sample of samples) strictEqual(validateMapleCloudEvent(sample).event.id, sample.id) + }) +}) diff --git a/docs/gitlab-event-projectors.md b/docs/gitlab-event-projectors.md new file mode 100644 index 000000000..c90b9dce9 --- /dev/null +++ b/docs/gitlab-event-projectors.md @@ -0,0 +1,168 @@ +# GitLab event projectors + +Status: version-1 producer contracts for the Maple Local eventing outbox. + +This document describes the projectors registered by the Local runtime. They +normalize the already bounded OTLP fields emitted by the GitLab repository-event +receiver. They do not call GitLab, create Matrix rooms, send Matrix messages, +or acknowledge a downstream consumer. + +## Input compatibility + +The production receiver exports the normalized event name as the OTLP log +attribute `event.name`. The original issue-created vertical also accepts the +OTLP LogRecord `eventName` field through the existing `signal:event.name` +fixture. New projectors prefer `attribute:event.name` and accept the signal +field as a compatibility fallback. + +All project and object IDs are retained as decimal strings because OTLP int64 +values are represented that way in the normalized signal model. Project IDs +and object IDs must be positive. Optional numeric durations and stage counts +must be non-negative. Strings are trimmed, rejected when blank, and limited to +4 KiB by the projector boundary. Raw webhook bodies, variables, URLs, secrets, +and confidential text are not projected. + +## Version-1 contracts + +### `gitlab.project.lifecycle@1` + +Input `event.name` values and normalized actions are: + +| Input | `action` | +| -------------------------- | -------------------- | +| `project_create` | `created` | +| `project_destroy` | `destroyed` | +| `project_rename` | `renamed` | +| `project_transfer` | `transferred` | +| `project_update` | `updated` | +| `project_archive` | `archived` | +| `project_unarchive` | `unarchived` | +| `project_deletion_request` | `deletion_requested` | + +Required fields are `gitlab.project.id`, `gitlab.project.path`, and +`event.name`. `gitlab.project.old_path`, `gitlab.actor.id`, +`gitlab.actor.name`, `gitlab.event.result`, and resource `service.name` are +optional. + +The output type is `dev.maple.gitlab.project.lifecycle.v1`, with data shaped as: + +```json +{ + "project": { "id": "42", "path": "rdev/maple", "oldPath": "rdev/old-maple" }, + "action": "renamed", + "sourceEvent": "project_rename", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" +} +``` + +### `gitlab.merge-request.lifecycle@1` + +The source event must match `merge_request_`, where `` is a +bounded lowercase GitLab action token. The required fields are +`gitlab.project.id`, `gitlab.project.path`, `gitlab.merge_request.iid`, and +`event.name`. Source branch, target branch, merge commit, actor, result, and +resource service name are optional. + +The output type is `dev.maple.gitlab.merge-request.lifecycle.v1`. Its subject +is `/-/merge_requests/` and its data has this shape: + +```json +{ + "project": { "id": "42", "path": "rdev/maple" }, + "mergeRequest": { + "iid": "7", + "action": "open", + "sourceBranch": "feature/maple", + "targetBranch": "main", + "commit": "abc123" + }, + "sourceEvent": "merge_request_open", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" +} +``` + +### `gitlab.pipeline.completed@1` + +The source event must be `ci_pipeline_completed`. The required fields are +`gitlab.project.id`, `gitlab.project.path`, `gitlab.ci.pipeline.id`, +`gitlab.ci.pipeline.status`, and `event.name`. Status must be one of +`success`, `failed`, `canceled`, or `skipped`. Pipeline IID, name, source, +detailed status, ref, revision, durations, stage count, actor, result, and +resource service name are optional. + +The output type is `dev.maple.gitlab.pipeline.completed.v1`. Its subject is +`/-/pipelines/`: + +```json +{ + "project": { "id": "42", "path": "rdev/maple" }, + "pipeline": { + "id": "900", + "iid": "12", + "status": "failed", + "name": "Maple CI", + "source": "push", + "detailedStatus": "failed", + "ref": "main", + "sha": "deadbeef", + "durationMs": "63000", + "queuedDurationMs": "10000", + "stageCount": "3" + }, + "sourceEvent": "ci_pipeline_completed", + "actor": { "id": "9", "name": "rdev" }, + "result": "failure", + "serviceName": "gitlab-repository-events" +} +``` + +## Complete CloudEvent fixtures + +These examples use the same source, occurrence IDs, projection IDs, and +revision as the compatibility tests. The IDs are the canonical Maple v1 +identity, so retrying the same source occurrence produces the same event ID. +The complete machine-readable set is checked in at +`apps/cli/test/fixtures/gitlab-projectors.v1.json`. + +```json +{ + "specversion": "1.0", + "id": "sha256:0fac0a375c6f8a05fb5ec1a751cf9e3b55282f56c0d135bc8a8c50c6a2a7559f", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.project.lifecycle.v1", + "subject": "rdev/maple", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-project-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-project-renamed", + "projectionrevision": 1, + "projectorid": "gitlab.project.lifecycle", + "projectorversion": 1, + "data": { + "project": { "id": "42", "path": "rdev/maple", "oldPath": "rdev/old-maple" }, + "action": "renamed", + "sourceEvent": "project_rename", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } +} +``` + +The merge-request fixture uses +`sha256:09992cc6804e056fb2a037b6c9db05c432c4733e229c963675a15a47445f745b` +with occurrence ID `mr-event-7` and projection ID `gitlab-mr-opened`. The +pipeline fixture uses +`sha256:f2bab7e420aa6b97d65f0478d16384b45d3bf61fb9ad9c111c614279cb42e24e` +with occurrence ID `pipeline-event-900` and projection ID +`gitlab-pipeline-completed`; its status is `failed` and its normalized result +is `failure`. + +Downstream Matrix delivery may use these stable event IDs for idempotent +transaction IDs, but that consumer protocol is deliberately outside these +projectors. From 581f22c2a1f824f92d8d932c592da15fb98579a5 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 13 Aug 2026 20:08:06 -0400 Subject: [PATCH 07/12] feat(cli): add durable event consumers --- apps/cli/src/server/eventing/consumer-auth.ts | 35 ++ apps/cli/src/server/eventing/control-store.ts | 438 +++++++++++++++++- apps/cli/src/server/eventing/runtime.ts | 21 + apps/cli/src/server/serve.ts | 176 ++++++- .../test/local-eventing-consumer-auth.test.ts | 48 ++ .../test/local-eventing-control-store.test.ts | 217 ++++++++- apps/cli/test/local-eventing-ingest.test.ts | 119 +++++ docs/local-event-consumers.md | 129 ++++++ docs/signal-to-event-projection.md | 13 +- 9 files changed, 1175 insertions(+), 21 deletions(-) create mode 100644 apps/cli/src/server/eventing/consumer-auth.ts create mode 100644 apps/cli/test/local-eventing-consumer-auth.test.ts create mode 100644 docs/local-event-consumers.md diff --git a/apps/cli/src/server/eventing/consumer-auth.ts b/apps/cli/src/server/eventing/consumer-auth.ts new file mode 100644 index 000000000..3751a4b5c --- /dev/null +++ b/apps/cli/src/server/eventing/consumer-auth.ts @@ -0,0 +1,35 @@ +import { randomBytes, timingSafeEqual } from "node:crypto" +import { lstatSync, readFileSync } from "node:fs" +import { resolve } from "node:path" +import { durableWrite } from "../durable-files" + +const TOKEN_BYTES = 32 + +export const eventConsumerTokenPath = (dataDir: string): string => `${resolve(dataDir)}.event-consumer-token` + +const readRealFile = (path: string): string => { + const stat = lstatSync(path) + if (stat.isSymbolicLink() || !stat.isFile()) + throw new Error(`event consumer token is not a real file: ${path}`) + return readFileSync(path, "utf8") +} + +export const ensureEventConsumerToken = async (dataDir: string): Promise => { + const path = eventConsumerTokenPath(dataDir) + try { + readRealFile(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + await durableWrite(path, `${randomBytes(TOKEN_BYTES).toString("hex")}\n`) + } + const token = readRealFile(path).trim() + if (!/^[0-9a-f]{64}$/.test(token)) throw new Error("event consumer token is malformed") + return token +} + +export const eventConsumerTokenMatches = (expected: string, supplied: string | null): boolean => { + if (supplied === null) return false + const left = Buffer.from(expected) + const right = Buffer.from(supplied) + return left.length === right.length && timingSafeEqual(left, right) +} diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts index 827a02d43..883163787 100644 --- a/apps/cli/src/server/eventing/control-store.ts +++ b/apps/cli/src/server/eventing/control-store.ts @@ -1,4 +1,5 @@ import { constants as sqliteConstants, Database } from "bun:sqlite" +import { createHash, randomBytes, timingSafeEqual } from "node:crypto" import { chmodSync, existsSync, lstatSync, readFileSync } from "node:fs" import { join, resolve } from "node:path" import { pathToFileURL } from "node:url" @@ -15,12 +16,13 @@ import { import { Schema } from "effect" import { durableWrite, ensurePrivateDirectory } from "../durable-files" -const CONTROL_SCHEMA_VERSION = 1 +const CONTROL_SCHEMA_VERSION = 2 const CONTROL_DIRECTORY = "control" const CONTROL_DATABASE = "eventing.sqlite" const MAX_FAILURES_PER_TENANT = 10_000 export const DEFAULT_MAX_OUTBOX_EVENTS = 10_000 export const DEFAULT_MAX_OUTBOX_BYTES = 256 * 1024 * 1024 +export const DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS = 1_000 export const eventingControlDirectory = (dataDir: string): string => join(resolve(dataDir), CONTROL_DIRECTORY) export const eventingControlPath = (dataDir: string): string => @@ -87,7 +89,59 @@ CREATE UNIQUE INDEX projection_failures_occurrence ON projection_failures (tenant_id, projection_id, projection_revision, occurrence_id) WHERE occurrence_id IS NOT NULL; -PRAGMA user_version = 1; +CREATE TABLE event_consumers ( + consumer_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + active INTEGER NOT NULL CHECK (active IN (0, 1)), + last_acked_sequence INTEGER NOT NULL CHECK (last_acked_sequence >= 0), + lease_token_hash TEXT, + lease_expires_at TEXT, + claimed_through_sequence INTEGER CHECK (claimed_through_sequence > 0), + registered_at TEXT NOT NULL, + disabled_at TEXT, + CHECK ( + (active = 1 AND disabled_at IS NULL) OR + (active = 0 AND disabled_at IS NOT NULL) + ), + CHECK ( + (lease_token_hash IS NULL AND lease_expires_at IS NULL AND claimed_through_sequence IS NULL) OR + (lease_token_hash IS NOT NULL AND lease_expires_at IS NOT NULL AND claimed_through_sequence IS NOT NULL) + ), + CHECK (claimed_through_sequence IS NULL OR claimed_through_sequence > last_acked_sequence) +) STRICT; + +CREATE INDEX event_consumers_tenant_active_ack + ON event_consumers (tenant_id, active, last_acked_sequence); + +PRAGMA user_version = 2; +` + +const MIGRATE_SCHEMA_1_TO_2 = ` +CREATE TABLE event_consumers ( + consumer_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + active INTEGER NOT NULL CHECK (active IN (0, 1)), + last_acked_sequence INTEGER NOT NULL CHECK (last_acked_sequence >= 0), + lease_token_hash TEXT, + lease_expires_at TEXT, + claimed_through_sequence INTEGER CHECK (claimed_through_sequence > 0), + registered_at TEXT NOT NULL, + disabled_at TEXT, + CHECK ( + (active = 1 AND disabled_at IS NULL) OR + (active = 0 AND disabled_at IS NOT NULL) + ), + CHECK ( + (lease_token_hash IS NULL AND lease_expires_at IS NULL AND claimed_through_sequence IS NULL) OR + (lease_token_hash IS NOT NULL AND lease_expires_at IS NOT NULL AND claimed_through_sequence IS NOT NULL) + ), + CHECK (claimed_through_sequence IS NULL OR claimed_through_sequence > last_acked_sequence) +) STRICT; + +CREATE INDEX event_consumers_tenant_active_ack + ON event_consumers (tenant_id, active, last_acked_sequence); + +PRAGMA user_version = 2; ` interface UserVersionRow { @@ -134,6 +188,26 @@ interface OutboxUsageRow { readonly bytes: number | bigint } +interface SequenceRow { + readonly sequence: number | bigint | null +} + +interface ConsumerRow { + readonly consumer_id: string + readonly tenant_id: string + readonly active: number | bigint + readonly last_acked_sequence: number | bigint + readonly lease_token_hash: string | null + readonly lease_expires_at: string | null + readonly claimed_through_sequence: number | bigint | null + readonly registered_at: string + readonly disabled_at: string | null +} + +interface EventIdRow { + readonly event_id: string +} + export interface StageEventsResult { readonly inserted: number readonly deduplicated: number @@ -151,6 +225,13 @@ export interface EventingControlSnapshotValidation { export interface LocalEventingControlLimits { readonly maxOutboxEvents: number readonly maxOutboxBytes: number + readonly retainAcknowledgedReadyEvents?: number +} + +interface ResolvedLocalEventingControlLimits { + readonly maxOutboxEvents: number + readonly maxOutboxBytes: number + readonly retainAcknowledgedReadyEvents: number } export interface EventingOutboxRecord { @@ -165,6 +246,37 @@ export interface EventingOutboxPage { readonly nextCursor: number | null } +export type EventConsumerStart = "beginning" | "latest" + +export interface EventConsumer { + readonly consumerId: string + readonly tenantId: string + readonly active: boolean + readonly lastAcknowledgedSequence: number + readonly leaseExpiresAt: string | null + readonly claimedThroughSequence: number | null + readonly registeredAt: string + readonly disabledAt: string | null +} + +export interface EventConsumerClaim { + readonly consumerId: string + readonly leaseToken: string | null + readonly leaseExpiresAt: string | null + readonly throughSequence: number | null + readonly events: readonly EventingOutboxRecord[] +} + +export interface EventConsumerAcknowledgement { + readonly consumerId: string + readonly acknowledgedThrough: number + readonly prunedEvents: number +} + +export class EventConsumerInputError extends Error {} +export class EventConsumerNotFoundError extends Error {} +export class EventConsumerConflictError extends Error {} + const asNumber = (value: number | bigint): number => { const number = Number(value) if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) @@ -208,23 +320,30 @@ const checkpointWal = (db: Database): void => { ) } -const validateLimits = (limits: LocalEventingControlLimits): LocalEventingControlLimits => { +const validateLimits = (limits: LocalEventingControlLimits): ResolvedLocalEventingControlLimits => { if (!Number.isSafeInteger(limits.maxOutboxEvents) || limits.maxOutboxEvents < 1) throw new Error("maxOutboxEvents must be a positive safe integer") if (!Number.isSafeInteger(limits.maxOutboxBytes) || limits.maxOutboxBytes < 1) throw new Error("maxOutboxBytes must be a positive safe integer") - return limits + const retainAcknowledgedReadyEvents = + limits.retainAcknowledgedReadyEvents ?? DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS + if (!Number.isSafeInteger(retainAcknowledgedReadyEvents) || retainAcknowledgedReadyEvents < 0) + throw new Error("retainAcknowledgedReadyEvents must be a non-negative safe integer") + return { ...limits, retainAcknowledgedReadyEvents } } -const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation => { +const validateOpenDatabase = ( + db: Database, + acceptedSchemaVersions: readonly number[] = [CONTROL_SCHEMA_VERSION], +): EventingControlSnapshotValidation => { const quick = db.query("PRAGMA quick_check").get() if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) const version = db.query("PRAGMA user_version").get() if (!version) throw new Error("eventing control database has no schema version") const schemaVersion = asNumber(version.user_version) - if (schemaVersion !== CONTROL_SCHEMA_VERSION) + if (!acceptedSchemaVersions.includes(schemaVersion)) throw new Error( - `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + `unsupported eventing control schema ${schemaVersion}; expected ${acceptedSchemaVersions.join(" or ")}`, ) const count = (where: string): number => { const row = db.query(`SELECT count(*) AS count FROM outbox_events ${where}`).get() @@ -250,6 +369,20 @@ const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation = if (!invalidReadiness) throw new Error("eventing readiness validation query returned no row") if (asNumber(invalidReadiness.count) !== 0) throw new Error("eventing control database has inconsistent outbox readiness state") + if (schemaVersion >= 2) { + const consumers = db + .query, []>( + "SELECT lease_expires_at, registered_at, disabled_at FROM event_consumers", + ) + .all() + for (const consumer of consumers) { + canonicalInstant(consumer.registered_at, "event consumer registeredAt") + if (consumer.lease_expires_at !== null) + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") + if (consumer.disabled_at !== null) + canonicalInstant(consumer.disabled_at, "event consumer disabledAt") + } + } return { schemaVersion, projectionRevisions: asNumber(revisions.count), @@ -259,12 +392,51 @@ const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation = } } +const CONSUMER_ID = /^[a-z][a-z0-9._-]{0,63}$/ +const LEASE_TOKEN = /^[0-9a-f]{64}$/ + +const validateConsumerId = (consumerId: string): string => { + if (!CONSUMER_ID.test(consumerId)) + throw new EventConsumerInputError( + "consumerId must start with a lowercase letter and contain at most 64 lowercase letters, digits, dots, underscores, or hyphens", + ) + return consumerId +} + +const canonicalInstant = (value: string, label: string): number => { + const milliseconds = Date.parse(value) + if (Number.isNaN(milliseconds) || new Date(milliseconds).toISOString() !== value) + throw new EventConsumerInputError(`${label} must be canonical ISO-8601`) + return milliseconds +} + +const tokenHash = (token: string): string => createHash("sha256").update(token).digest("hex") + +const tokenHashMatches = (expected: string, token: string): boolean => { + if (!LEASE_TOKEN.test(token)) return false + const left = Buffer.from(expected, "hex") + const right = Buffer.from(tokenHash(token), "hex") + return left.length === right.length && timingSafeEqual(left, right) +} + +const decodeConsumer = (row: ConsumerRow): EventConsumer => ({ + consumerId: row.consumer_id, + tenantId: row.tenant_id, + active: asNumber(row.active) === 1, + lastAcknowledgedSequence: asNumber(row.last_acked_sequence), + leaseExpiresAt: row.lease_expires_at, + claimedThroughSequence: + row.claimed_through_sequence === null ? null : asNumber(row.claimed_through_sequence), + registeredAt: row.registered_at, + disabledAt: row.disabled_at, +}) + export class LocalEventingControlStore { readonly #db: Database - readonly #limits: LocalEventingControlLimits + readonly #limits: ResolvedLocalEventingControlLimits readonly path: string - private constructor(path: string, db: Database, limits: LocalEventingControlLimits) { + private constructor(path: string, db: Database, limits: ResolvedLocalEventingControlLimits) { this.path = path this.#db = db this.#limits = limits @@ -275,9 +447,10 @@ export class LocalEventingControlStore { limits: LocalEventingControlLimits = { maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, + retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, }, ): Promise { - validateLimits(limits) + const validatedLimits = validateLimits(limits) const directory = eventingControlDirectory(dataDir) await ensurePrivateDirectory(directory) const path = eventingControlPath(dataDir) @@ -291,13 +464,14 @@ export class LocalEventingControlStore { if (!version) throw new Error("eventing control database has no schema version") const schemaVersion = asNumber(version.user_version) if (schemaVersion === 0) db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() + else if (schemaVersion === 1) db.transaction(() => db.exec(MIGRATE_SCHEMA_1_TO_2)).exclusive() else if (schemaVersion !== CONTROL_SCHEMA_VERSION) throw new Error( `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, ) chmodSync(path, 0o600) validateOpenDatabase(db) - return new LocalEventingControlStore(path, db, limits) + return new LocalEventingControlStore(path, db, validatedLimits) } catch (error) { db.close() throw error @@ -507,6 +681,246 @@ export class LocalEventingControlStore { return this.#listOutbox("staged", limit, after) } + listConsumers(tenantId: string): readonly EventConsumer[] { + return this.#db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? + ORDER BY consumer_id`, + ) + .all(tenantId) + .map(decodeConsumer) + } + + registerConsumer( + tenantId: string, + consumerId: string, + startAt: EventConsumerStart, + registeredAt = new Date().toISOString(), + ): EventConsumer { + validateConsumerId(consumerId) + if (startAt !== "beginning" && startAt !== "latest") + throw new EventConsumerInputError("startAt must be beginning or latest") + canonicalInstant(registeredAt, "event consumer registeredAt") + return this.#db + .transaction(() => { + const existing = this.#consumer(tenantId, consumerId) + if (existing) + throw new EventConsumerConflictError(`event consumer already exists: ${consumerId}`) + const boundary = this.#db + .query( + startAt === "latest" + ? `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ?` + : `SELECT min(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ?`, + ) + .get(tenantId) + const sequence = boundary?.sequence == null ? 0 : asNumber(boundary.sequence) + const lastAcknowledged = startAt === "beginning" ? Math.max(0, sequence - 1) : sequence + this.#db.run( + "INSERT INTO event_consumers (consumer_id, tenant_id, active, last_acked_sequence, registered_at) VALUES (?, ?, 1, ?, ?)", + [consumerId, tenantId, lastAcknowledged, registeredAt], + ) + return decodeConsumer(this.#consumer(tenantId, consumerId)!) + }) + .immediate() + } + + disableConsumer( + tenantId: string, + consumerId: string, + disabledAt = new Date().toISOString(), + ): EventConsumer { + validateConsumerId(consumerId) + canonicalInstant(disabledAt, "event consumer disabledAt") + return this.#db + .transaction(() => { + const existing = this.#consumer(tenantId, consumerId) + if (!existing) throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(existing.active) === 0) return decodeConsumer(existing) + this.#db.run( + `UPDATE event_consumers + SET active = 0, lease_token_hash = NULL, lease_expires_at = NULL, + claimed_through_sequence = NULL, disabled_at = ? + WHERE tenant_id = ? AND consumer_id = ?`, + [disabledAt, tenantId, consumerId], + ) + this.#pruneAcknowledgedReady(tenantId) + return decodeConsumer(this.#consumer(tenantId, consumerId)!) + }) + .immediate() + } + + claimReady( + tenantId: string, + consumerId: string, + limit: number, + leaseSeconds: number, + now = new Date().toISOString(), + ): EventConsumerClaim { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new EventConsumerInputError("claim limit must be between 1 and 1000") + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) + throw new EventConsumerInputError("leaseSeconds must be between 5 and 300") + const nowMilliseconds = canonicalInstant(now, "claim time") + return this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_expires_at !== null && + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > + nowMilliseconds + ) + throw new EventConsumerConflictError( + `event consumer already has an active lease: ${consumerId}`, + ) + + const rows = this.#db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(tenantId, asNumber(consumer.last_acked_sequence), limit) + if (rows.length === 0) { + this.#db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", + [tenantId, consumerId], + ) + return { + consumerId, + leaseToken: null, + leaseExpiresAt: null, + throughSequence: null, + events: [], + } + } + + const leaseToken = randomBytes(32).toString("hex") + const leaseExpiresAt = new Date(nowMilliseconds + leaseSeconds * 1_000).toISOString() + const throughSequence = asNumber(rows.at(-1)!.sequence) + this.#db.run( + `UPDATE event_consumers + SET lease_token_hash = ?, lease_expires_at = ?, claimed_through_sequence = ? + WHERE tenant_id = ? AND consumer_id = ?`, + [tokenHash(leaseToken), leaseExpiresAt, throughSequence, tenantId, consumerId], + ) + return { + consumerId, + leaseToken, + leaseExpiresAt, + throughSequence, + events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })), + } + }) + .immediate() + } + + acknowledgeClaim( + tenantId: string, + consumerId: string, + leaseToken: string, + throughSequence: number, + now = new Date().toISOString(), + ): EventConsumerAcknowledgement { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) + throw new EventConsumerInputError("throughSequence must be a positive safe integer") + const nowMilliseconds = canonicalInstant(now, "acknowledgement time") + return this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_token_hash === null || + consumer.lease_expires_at === null || + consumer.claimed_through_sequence === null + ) + throw new EventConsumerConflictError(`event consumer has no active lease: ${consumerId}`) + if ( + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") <= + nowMilliseconds + ) + throw new EventConsumerConflictError(`event consumer lease has expired: ${consumerId}`) + if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) + throw new EventConsumerConflictError("event consumer lease token does not match") + const claimedThrough = asNumber(consumer.claimed_through_sequence) + if (throughSequence !== claimedThrough) + throw new EventConsumerConflictError( + `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, + ) + this.#db.run( + `UPDATE event_consumers + SET last_acked_sequence = ?, lease_token_hash = NULL, lease_expires_at = NULL, + claimed_through_sequence = NULL + WHERE tenant_id = ? AND consumer_id = ?`, + [throughSequence, tenantId, consumerId], + ) + return { + consumerId, + acknowledgedThrough: throughSequence, + prunedEvents: this.#pruneAcknowledgedReady(tenantId), + } + }) + .immediate() + } + + #consumer(tenantId: string, consumerId: string): ConsumerRow | null { + return this.#db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? AND consumer_id = ?`, + ) + .get(tenantId, consumerId) + } + + #pruneAcknowledgedReady(tenantId: string): number { + const boundary = this.#db + .query( + "SELECT min(last_acked_sequence) AS sequence FROM event_consumers WHERE tenant_id = ? AND active = 1", + ) + .get(tenantId) + if (boundary?.sequence == null) return 0 + const rows = this.#db + .query( + `SELECT readiness.event_id + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND readiness.sequence <= ? + ORDER BY readiness.sequence`, + ) + .all(tenantId, asNumber(boundary.sequence)) + const pruneCount = Math.max(0, rows.length - this.#limits.retainAcknowledgedReadyEvents) + for (const { event_id } of rows.slice(0, pruneCount)) { + this.#db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [event_id]) + this.#db.run("DELETE FROM outbox_events WHERE event_id = ? AND state = 'ready'", [event_id]) + } + return pruneCount + } + outboxCapacity(): LocalEventingControlLimits & { readonly currentEvents: number readonly currentBytes: number @@ -572,7 +986,7 @@ export class LocalEventingControlStore { const db = new Database(uri, sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI) try { configure(db) - return validateOpenDatabase(db) + return validateOpenDatabase(db, [1, CONTROL_SCHEMA_VERSION]) } finally { db.close(true) } diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts index 2b7085392..b3542abe3 100644 --- a/apps/cli/src/server/eventing/runtime.ts +++ b/apps/cli/src/server/eventing/runtime.ts @@ -10,6 +10,7 @@ import { } from "@maple/eventing-core" import { Schema } from "effect" import { LocalEventingControlStore } from "./control-store" +import type { EventConsumerStart } from "./control-store" import { registerGitLabProjectors } from "./gitlab-projectors" import { OTLP_LOG_ADAPTER } from "./otlp" @@ -129,6 +130,26 @@ export class LocalEventingRuntime { return this.#store.listStaged(limit, after) } + listConsumers() { + return this.#store.listConsumers(TENANT_ID) + } + + registerConsumer(consumerId: string, startAt: EventConsumerStart) { + return this.#store.registerConsumer(TENANT_ID, consumerId, startAt) + } + + disableConsumer(consumerId: string) { + return this.#store.disableConsumer(TENANT_ID, consumerId) + } + + claimReady(consumerId: string, limit: number, leaseSeconds: number) { + return this.#store.claimReady(TENANT_ID, consumerId, limit, leaseSeconds) + } + + acknowledgeClaim(consumerId: string, leaseToken: string, throughSequence: number) { + return this.#store.acknowledgeClaim(TENANT_ID, consumerId, leaseToken, throughSequence) + } + health() { return { activeProjections: this.listActive().length, diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 6592a2782..f903b0073 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -17,7 +17,14 @@ import { rawTelemetryTtlStatements, } from "./chdb" import { buildInsertStatements } from "./inserts" -import { eventingControlSnapshotPath, LocalEventingControlStore } from "./eventing/control-store" +import { + eventingControlSnapshotPath, + EventConsumerConflictError, + EventConsumerInputError, + EventConsumerNotFoundError, + LocalEventingControlStore, +} from "./eventing/control-store" +import { ensureEventConsumerToken, eventConsumerTokenMatches } from "./eventing/consumer-auth" import { LocalEventingRuntime } from "./eventing/runtime" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { @@ -631,6 +638,7 @@ const handleRetirement = async ( const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i const MAX_CHECKPOINT_BODY_BYTES = 4 * 1024 const MAX_PROJECTION_BODY_BYTES = 512 * 1024 +const MAX_CONSUMER_BODY_BYTES = 16 * 1024 /** Typed, authenticated replacement for sending BACKUP through /local/query. */ const handleCheckpointBackup = async ( @@ -712,6 +720,150 @@ const handleProjectionActivation = async ( } } +const eventConsumerErrorResponse = (error: unknown): Response => { + const message = error instanceof Error ? error.message : String(error) + if (error instanceof EventConsumerNotFoundError) return text(message, 404) + if (error instanceof EventConsumerConflictError) return text(message, 409) + if (error instanceof EventConsumerInputError) return text(message, 400) + return text(`event consumer operation failed: ${message}`, 500) +} + +const handleConsumerRegistration = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + maintenanceToken: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if ( + Object.keys(record).sort().join(",") !== "consumerId,startAt" || + typeof record.consumerId !== "string" || + (record.startAt !== "beginning" && record.startAt !== "latest") + ) + return text("invalid event consumer registration fields", 400) + const consumerId = record.consumerId as string + const startAt = record.startAt as "beginning" | "latest" + return admitted(gate, async () => { + try { + return json(eventing.registerConsumer(consumerId, startAt), 201) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerDisable = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + maintenanceToken: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if (Object.keys(record).join(",") !== "consumerId" || typeof record.consumerId !== "string") + return text("invalid event consumer disable fields", 400) + return admitted(gate, async () => { + try { + return json(eventing.disableConsumer(record.consumerId as string)) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerClaim = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + consumerToken: string, + req: Request, +): Promise => { + if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) + return text("event consumer authorization required", 403) + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if ( + Object.keys(record).sort().join(",") !== "consumerId,leaseSeconds,limit" || + typeof record.consumerId !== "string" || + typeof record.limit !== "number" || + typeof record.leaseSeconds !== "number" + ) + return text("invalid event consumer claim fields", 400) + return admitted(gate, async () => { + try { + return json( + eventing.claimReady( + record.consumerId as string, + record.limit as number, + record.leaseSeconds as number, + ), + ) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerAcknowledgement = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + consumerToken: string, + req: Request, +): Promise => { + if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) + return text("event consumer authorization required", 403) + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if ( + Object.keys(record).sort().join(",") !== "consumerId,leaseToken,throughSequence" || + typeof record.consumerId !== "string" || + typeof record.leaseToken !== "string" || + typeof record.throughSequence !== "number" + ) + return text("invalid event consumer acknowledgement fields", 400) + return admitted(gate, async () => { + try { + return json( + eventing.acknowledgeClaim( + record.consumerId as string, + record.leaseToken as string, + record.throughSequence as number, + ), + ) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + const handleEventingRead = ( eventing: LocalEventingRuntime, token: string, @@ -722,6 +874,7 @@ const handleEventingRead = ( if (unauthorized) return unauthorized if (url.pathname === "/local/eventing/health") return json(eventing.health()) if (url.pathname === "/local/eventing/projections") return json(eventing.listActive()) + if (url.pathname === "/local/eventing/consumers") return json(eventing.listConsumers()) if (url.pathname === "/local/eventing/outbox") { const rawLimit = url.searchParams.get("limit") const limit = rawLimit === null ? 100 : Number(rawLimit) @@ -750,6 +903,7 @@ const makeFetch = authority: RetiredDayAuthority, gate: RequestQuiescenceGate, maintenanceToken: string, + consumerToken: string, controlStore: LocalEventingControlStore, eventing: LocalEventingRuntime, ) => @@ -791,6 +945,14 @@ const makeFetch = ) if (url.pathname === "/local/eventing/projections") return respond(await handleProjectionActivation(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/consumers") + return respond(await handleConsumerRegistration(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/consumers/disable") + return respond(await handleConsumerDisable(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/claims") + return respond(await handleConsumerClaim(eventing, gate, consumerToken, req)) + if (url.pathname === "/local/eventing/acks") + return respond(await handleConsumerAcknowledgement(eventing, gate, consumerToken, req)) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } @@ -906,6 +1068,13 @@ export const startServer = ( message: `failed to load maintenance token: ${error instanceof Error ? error.message : String(error)}`, }), }) + const consumerToken = yield* Effect.tryPromise({ + try: () => ensureEventConsumerToken(options.dataDir), + catch: (error) => + new ChdbError({ + message: `failed to load event consumer token: ${error instanceof Error ? error.message : String(error)}`, + }), + }) const gate = new RequestQuiescenceGate() // A dedicated runtime carrying the OTel tracer for per-request spans: the // Bun.serve handler runs outside Effect, so each request's span effect is @@ -929,6 +1098,7 @@ export const startServer = ( authority, gate, maintenanceToken, + consumerToken, controlStore, eventing, ), @@ -946,6 +1116,10 @@ export const startServer = ( }) export const __testables = { + handleConsumerAcknowledgement, + handleConsumerClaim, + handleConsumerDisable, + handleConsumerRegistration, handleCheckpointBackup, handleEventingRead, handleProjectionActivation, diff --git a/apps/cli/test/local-eventing-consumer-auth.test.ts b/apps/cli/test/local-eventing-consumer-auth.test.ts new file mode 100644 index 000000000..6e264bdcd --- /dev/null +++ b/apps/cli/test/local-eventing-consumer-auth.test.ts @@ -0,0 +1,48 @@ +import { strictEqual } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import { + ensureEventConsumerToken, + eventConsumerTokenMatches, + eventConsumerTokenPath, +} from "../src/server/eventing/consumer-auth" + +describe("local event consumer authorization", () => { + it("creates a stable private token separate from the data directory", async () => { + const parent = mkdtempSync(join(tmpdir(), "maple-event-consumer-auth-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir) + try { + const first = await ensureEventConsumerToken(dataDir) + const second = await ensureEventConsumerToken(dataDir) + strictEqual(first.length, 64) + strictEqual(second, first) + strictEqual(statSync(eventConsumerTokenPath(dataDir)).mode & 0o777, 0o600) + strictEqual(eventConsumerTokenMatches(first, first), true) + strictEqual(eventConsumerTokenMatches(first, `${first}0`), false) + strictEqual(eventConsumerTokenMatches(first, null), false) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) + + it("refuses a symlink in place of the token", async () => { + const parent = mkdtempSync(join(tmpdir(), "maple-event-consumer-auth-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir) + try { + symlinkSync(join(parent, "target"), eventConsumerTokenPath(dataDir)) + let message = "" + try { + await ensureEventConsumerToken(dataDir) + } catch (error) { + message = error instanceof Error ? error.message : String(error) + } + strictEqual(message.includes("not a real file"), true) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts index adad8ad36..8db98d366 100644 --- a/apps/cli/test/local-eventing-control-store.test.ts +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -1,4 +1,5 @@ import { deepStrictEqual, ok, rejects, strictEqual, throws } from "node:assert" +import { Database } from "bun:sqlite" import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" @@ -72,7 +73,7 @@ describe("LocalEventingControlStore", () => { store.saveProjection(projection({ revision: 3 })) deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) deepStrictEqual(store.validate(), { - schemaVersion: 1, + schemaVersion: 2, projectionRevisions: 3, projectionFailures: 0, stagedEvents: 0, @@ -131,7 +132,7 @@ describe("LocalEventingControlStore", () => { const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") const validation = await store.backupTo(snapshot) deepStrictEqual(validation, { - schemaVersion: 1, + schemaVersion: 2, projectionRevisions: 1, projectionFailures: 1, stagedEvents: 0, @@ -246,6 +247,218 @@ describe("LocalEventingControlStore", () => { } })) + it("migrates schema 1 in place and keeps schema-1 snapshots restorable", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.stageEvents([event()]) + store.markReady([event().id]) + store.close() + + const database = new Database(eventingControlPath(dataDir), { + readwrite: true, + strict: true, + safeIntegers: true, + }) + database.exec("DROP TABLE event_consumers") + database.exec("PRAGMA user_version = 1") + database.close(true) + + strictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(dataDir)).schemaVersion, + 1, + ) + store = await LocalEventingControlStore.open(dataDir) + try { + strictEqual(store.validate().schemaVersion, 2) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [event().id], + ) + deepStrictEqual(store.listConsumers("tenant-a"), []) + } finally { + store.close() + } + })) + + it("leases whole batches, redelivers after expiry, and rejects stale acknowledgements", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 10, + maxOutboxBytes: 1024 * 1024, + retainAcknowledgedReadyEvents: 0, + }) + try { + const second = event({ id: "event-2" }) + const third = event({ id: "event-3" }) + const staged = store.stageEvents([event(), second, third]) + store.markReady(staged.eventIds) + store.registerConsumer("tenant-a", "matrix", "beginning", "2026-08-13T12:00:00.000Z") + + const firstClaim = store.claimReady("tenant-a", "matrix", 2, 10, "2026-08-13T12:00:01.000Z") + strictEqual(firstClaim.leaseToken?.length, 64) + deepStrictEqual( + firstClaim.events.map(({ event }) => event.id), + [event().id, second.id], + ) + throws( + () => store.claimReady("tenant-a", "matrix", 2, 10, "2026-08-13T12:00:02.000Z"), + /active lease/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "matrix", + "0".repeat(64), + firstClaim.throughSequence!, + "2026-08-13T12:00:03.000Z", + ), + /token does not match/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "matrix", + firstClaim.leaseToken!, + firstClaim.events[0]!.sequence, + "2026-08-13T12:00:03.000Z", + ), + /complete claimed batch/, + ) + + const retry = store.claimReady("tenant-a", "matrix", 2, 10, "2026-08-13T12:00:12.000Z") + deepStrictEqual( + retry.events.map(({ event }) => event.id), + [event().id, second.id], + ) + strictEqual(retry.leaseToken === firstClaim.leaseToken, false) + deepStrictEqual( + store.acknowledgeClaim( + "tenant-a", + "matrix", + retry.leaseToken!, + retry.throughSequence!, + "2026-08-13T12:00:13.000Z", + ), + { + consumerId: "matrix", + acknowledgedThrough: retry.throughSequence, + prunedEvents: 2, + }, + ) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [third.id], + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "matrix", + firstClaim.leaseToken!, + firstClaim.throughSequence!, + "2026-08-13T12:00:14.000Z", + ), + /no active lease/, + ) + } finally { + store.close() + } + })) + + it("prunes only after every active consumer advances and never prunes staged events", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 10, + maxOutboxBytes: 1024 * 1024, + retainAcknowledgedReadyEvents: 0, + }) + try { + const second = event({ id: "event-2" }) + const third = event({ id: "event-3" }) + const stranded = event({ id: "event-staged" }) + const ready = store.stageEvents([event(), second, third]) + store.markReady(ready.eventIds) + store.stageEvents([stranded]) + store.registerConsumer("tenant-a", "matrix-a", "beginning") + store.registerConsumer("tenant-a", "matrix-b", "beginning") + + const fast = store.claimReady("tenant-a", "matrix-a", 3, 30) + strictEqual( + store.acknowledgeClaim("tenant-a", "matrix-a", fast.leaseToken!, fast.throughSequence!) + .prunedEvents, + 0, + ) + const slow = store.claimReady("tenant-a", "matrix-b", 2, 30) + strictEqual( + store.acknowledgeClaim("tenant-a", "matrix-b", slow.leaseToken!, slow.throughSequence!) + .prunedEvents, + 2, + ) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [third.id], + ) + store.disableConsumer("tenant-a", "matrix-b") + deepStrictEqual(store.listReady().events, []) + deepStrictEqual( + store.listStaged().events.map(({ event }) => event.id), + [stranded.id], + ) + } finally { + store.close() + } + })) + + it("starts latest consumers after backlog and checkpoints active leases", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.stageEvents([event()]) + store.markReady([event().id]) + const registered = store.registerConsumer( + "tenant-a", + "matrix", + "latest", + "2099-01-01T00:00:00.000Z", + ) + strictEqual(registered.lastAcknowledgedSequence, store.listReady().events[0]!.sequence) + deepStrictEqual( + store.claimReady("tenant-a", "matrix", 10, 300, "2099-01-01T00:00:01.000Z").events, + [], + ) + + const second = event({ id: "event-2" }) + store.stageEvents([second]) + store.markReady([second.id]) + const claim = store.claimReady("tenant-a", "matrix", 10, 300, "2099-01-01T00:00:02.000Z") + const snapshot = join(dataDir, "backups", "consumer", "control.sqlite") + await store.backupTo(snapshot) + store.close() + + const restored = join(dataDir, "restored-consumer") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + store = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual( + store.listConsumers("tenant-a")[0]?.claimedThroughSequence, + claim.throughSequence, + ) + strictEqual( + store.acknowledgeClaim( + "tenant-a", + "matrix", + claim.leaseToken!, + claim.throughSequence!, + "2099-01-01T00:00:03.000Z", + ).acknowledgedThrough, + claim.throughSequence, + ) + } finally { + store.close() + } + })) + it("refuses a symlink in place of the database", async () => withDataDir(async (dataDir) => { const controlPath = eventingControlPath(dataDir) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts index 5b64badfe..46e486f10 100644 --- a/apps/cli/test/local-eventing-ingest.test.ts +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -137,6 +137,125 @@ describe("Local eventing ingest seam", () => { await maintenance }) + it("separates consumer administration from claim and acknowledgement authorization", async () => { + const gate = new __testables.RequestQuiescenceGate() + const calls: string[] = [] + const eventing = { + registerConsumer: (consumerId: string, startAt: string) => { + calls.push(`register:${consumerId}:${startAt}`) + return { consumerId, active: true } + }, + disableConsumer: (consumerId: string) => { + calls.push(`disable:${consumerId}`) + return { consumerId, active: false } + }, + claimReady: (consumerId: string, limit: number, leaseSeconds: number) => { + calls.push(`claim:${consumerId}:${limit}:${leaseSeconds}`) + return { + consumerId, + leaseToken: "a".repeat(64), + throughSequence: 7, + events: [{ sequence: 7, event: { id: "event-7" } }], + } + }, + acknowledgeClaim: (consumerId: string, _leaseToken: string, throughSequence: number) => { + calls.push(`ack:${consumerId}:${throughSequence}`) + return { consumerId, acknowledgedThrough: throughSequence, prunedEvents: 0 } + }, + } + + const registration = await __testables.handleConsumerRegistration( + eventing as never, + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/consumers", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "matrix", startAt: "beginning" }), + }), + ) + strictEqual(registration.status, 201) + + const wrongClaimCredential = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "matrix", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(wrongClaimCredential.status, 403) + + const claim = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "matrix", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(claim.status, 200) + const claimed = (await claim.json()) as { leaseToken: string; throughSequence: number } + + const acknowledgement = await __testables.handleConsumerAcknowledgement( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/acks", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ + consumerId: "matrix", + leaseToken: claimed.leaseToken, + throughSequence: claimed.throughSequence, + }), + }), + ) + strictEqual(acknowledgement.status, 200) + deepStrictEqual(calls, ["register:matrix:beginning", "claim:matrix:10:30", "ack:matrix:7"]) + + let releaseMaintenance!: () => void + const maintenance = gate.exclusive( + () => + new Promise((resolve) => { + releaseMaintenance = resolve + }), + ) + await Promise.resolve() + const blockedClaim = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "matrix", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(blockedClaim.status, 503) + releaseMaintenance() + await maintenance + }) + it("isolates projection failures, stores telemetry, and makes sibling events ready", async () => { const order: string[] = [] const event = { id: "event-1" } diff --git a/docs/local-event-consumers.md b/docs/local-event-consumers.md new file mode 100644 index 000000000..f5e094d82 --- /dev/null +++ b/docs/local-event-consumers.md @@ -0,0 +1,129 @@ +# Maple Local event consumer protocol + +Status: version 1 durable downstream-consumer boundary for the Maple Local event outbox. + +This protocol lets a local bridge deliver ready Maple CloudEvents without destructive reads or a +second delivery database. It is intentionally transport-neutral: Maple does not send Matrix events, +store Matrix credentials, or choose rooms. + +## Credentials + +Maple creates two independent 32-byte hexadecimal credentials beside the configured data directory: + +- `.maintenance-token` administers projection and consumer configuration. +- `.event-consumer-token` permits only claim and acknowledgement requests. + +Both files must be real regular files. The consumer token is sent in +`x-maple-event-consumer-token`; it does not grant access to projection configuration, outbox +inspection, checkpoints, or retention controls. The existing maintenance token is sent in +`x-maple-maintenance-token` and cannot be substituted for the consumer token. + +## Consumer administration + +Consumer IDs match `^[a-z][a-z0-9._-]{0,63}$` and are unique. Disabled IDs remain reserved so an +operator cannot accidentally replace one consumer's durable position with an unrelated process. + +Register a consumer with the maintenance credential: + +```http +POST /local/eventing/consumers +Content-Type: application/json +X-Maple-Maintenance-Token: + +{"consumerId":"matrix","startAt":"beginning"} +``` + +`startAt` is exact: + +- `beginning` starts immediately before the earliest ready event still retained for the tenant. +- `latest` atomically skips every ready event visible at registration and receives later events. + +Successful registration returns `201` and the consumer record. Reusing any existing or disabled ID +returns `409`. `GET /local/eventing/consumers` lists records under maintenance authorization. + +Disable a consumer explicitly: + +```http +POST /local/eventing/consumers/disable +Content-Type: application/json +X-Maple-Maintenance-Token: + +{"consumerId":"matrix"} +``` + +Disabling clears any active lease and removes that cursor from the retention quorum. It does not +delete the audit record or permit the ID to be reused. + +## Claim and acknowledgement + +Claim between 1 and 1,000 ready events for a lease of 5 through 300 seconds: + +```http +POST /local/eventing/claims +Content-Type: application/json +X-Maple-Event-Consumer-Token: + +{"consumerId":"matrix","limit":100,"leaseSeconds":60} +``` + +A non-empty response has this shape: + +```json +{ + "consumerId": "matrix", + "leaseToken": "<64 lowercase hexadecimal characters>", + "leaseExpiresAt": "2026-08-13T16:01:00.000Z", + "throughSequence": 42, + "events": [ + { + "sequence": 42, + "event": { + "specversion": "1.0", + "id": "sha256:...", + "type": "dev.maple.gitlab.pipeline.completed.v1" + }, + "stagedAt": "2026-08-13T16:00:00.000Z", + "readyAt": "2026-08-13T16:00:00.010Z" + } + ] +} +``` + +The real `event` member is the complete validated CloudEvent. An empty claim returns null lease +fields and an empty event array. Only a SHA-256 hash of the lease token is stored. A second claim +while the lease is live returns `409`; at or after expiry it returns the same unacknowledged prefix, +possibly with a new token. + +After every event in the claimed batch has been accepted by the downstream system, acknowledge the +exact `throughSequence` returned by the claim: + +```http +POST /local/eventing/acks +Content-Type: application/json +X-Maple-Event-Consumer-Token: + +{"consumerId":"matrix","leaseToken":"","throughSequence":42} +``` + +Partial, extended, expired, missing, and wrong-token acknowledgements return `409`. Success returns: + +```json +{ "consumerId": "matrix", "acknowledgedThrough": 42, "prunedEvents": 0 } +``` + +Claims are at-least-once. A bridge crash after a downstream send and before acknowledgement causes +re-delivery after lease expiry. A Matrix bridge must therefore derive its Matrix transaction ID from +the immutable Maple CloudEvent `id`; retrying the same transaction ID makes that ambiguity harmless. + +## Retention, capacity, and checkpoints + +Ready events are eligible for pruning only through the lowest acknowledged sequence among all active +consumers for the tenant. Maple retains the newest 1,000 otherwise-prunable ready events by default. +Disabled consumers do not block pruning; staged events are never pruned by consumer acknowledgement. +If no consumer is active, acknowledgement retention performs no deletion. + +The eventing control database migrates transactionally from schema 1 to schema 2 on open. Existing +schema-1 control snapshots remain valid and are migrated after restore. Consumer cursors and leases +are part of the same SQLite backup as projection and outbox state. Consumer mutations enter the +server admission gate, so checkpoint exclusivity cannot capture a half-applied claim or +acknowledgement. diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md index 2e7fb7478..208a5464f 100644 --- a/docs/signal-to-event-projection.md +++ b/docs/signal-to-event-projection.md @@ -886,18 +886,19 @@ FULL`. While ingest is quiesced, backup first completes and verifies a blocking Maple Local activates immutable revisions with authenticated `POST /local/eventing/projections`. The same maintenance credential protects -`GET /local/eventing/projections`, `/local/eventing/health`, and -`/local/eventing/outbox`. Ready records receive a separate, append-only +`GET /local/eventing/projections`, `/local/eventing/health`, +`/local/eventing/outbox`, and consumer administration. Ready records receive a separate, append-only readiness `sequence` on their first staged-to-ready transition; `?after=&limit=` therefore cannot skip an older staged event that is recovered after newer events were already read. `?state=staged` uses the original staging sequence for bounded inspection of records stranded before the chDB commit point. The Local store defaults to at most 10,000 events and 256 MiB of canonical event JSON. Staging fails closed with a retryable ingest error before -either cap can be exceeded. These endpoints are an operable inspection/recovery -surface, not yet a delivery protocol: durable consumer claim/acknowledgement and -deletion/retention begin with the first downstream consumer rather than being -implied by a destructive read API in this PR. +either cap can be exceeded. Inspection remains non-destructive. Named downstream +consumers use the separate [Maple Local event consumer protocol](./local-event-consumers.md) for +leased, at-least-once claims and exact whole-batch acknowledgement. Ready-event pruning advances only +through the slowest active consumer and retains a bounded acknowledged tail; staged events are never +pruned by delivery acknowledgement. Re-delivery is the safe recovery operation: it deduplicates the same staged event ID and promotes it only after the warehouse write succeeds. Maple never blindly From b1d504578f15ece55f9efda20bb085e70a97b7bb Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 13 Aug 2026 21:30:02 -0400 Subject: [PATCH 08/12] feat(eventing): expose source occurrence identity --- apps/cli/test/local-eventing-runtime.test.ts | 2 ++ .../schemas/cloud-event.v1.schema.json | 18 ++++++++++++++++++ packages/eventing-core/src/event.ts | 2 ++ packages/eventing-core/src/model.ts | 4 ++++ packages/eventing-core/src/registry.test.ts | 13 +++++++++++++ 5 files changed, 39 insertions(+) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts index a68559ea2..a415ffefe 100644 --- a/apps/cli/test/local-eventing-runtime.test.ts +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -196,6 +196,8 @@ describe("LocalEventingRuntime", () => { projectionrevision: 1, projectorid: "gitlab.issue.created", projectorversion: 1, + sourceoccurrenceid: "01K20GITLABISSUE42", + sourceidentityquality: "source", data: { project: { id: "7", path: "platform/maple" }, issue: { diff --git a/packages/eventing-core/schemas/cloud-event.v1.schema.json b/packages/eventing-core/schemas/cloud-event.v1.schema.json index adae790a5..081aa10c7 100644 --- a/packages/eventing-core/schemas/cloud-event.v1.schema.json +++ b/packages/eventing-core/schemas/cloud-event.v1.schema.json @@ -139,6 +139,24 @@ } ] }, + "sourceoccurrenceid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "sourceidentityquality": { + "type": "string", + "enum": ["source"] + }, "data": {} }, "required": [ diff --git a/packages/eventing-core/src/event.ts b/packages/eventing-core/src/event.ts index 6dd7f42e3..f21e92b43 100644 --- a/packages/eventing-core/src/event.ts +++ b/packages/eventing-core/src/event.ts @@ -142,6 +142,8 @@ export const makeCloudEvent = (input: { projectionrevision: input.projection.revision, projectorid: input.projectorId, projectorversion: input.projectorVersion, + sourceoccurrenceid: input.signal.occurrenceId, + sourceidentityquality: input.signal.identityQuality, data: input.data, }).event } diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts index b298e8b77..d74e57d83 100644 --- a/packages/eventing-core/src/model.ts +++ b/packages/eventing-core/src/model.ts @@ -234,6 +234,8 @@ export interface MapleCloudEvent { readonly projectionrevision: number readonly projectorid: string readonly projectorversion: number + readonly sourceoccurrenceid?: string + readonly sourceidentityquality?: "source" | "derived" | "none" readonly data: JsonValue } @@ -251,6 +253,8 @@ export const MapleCloudEventSchema = Schema.Struct({ projectionrevision: Schema.Int.check(Schema.isGreaterThan(0)), projectorid: NonEmptyIdentifier, projectorversion: Schema.Int.check(Schema.isGreaterThan(0)), + sourceoccurrenceid: Schema.optionalKey(NonEmptyIdentifier), + sourceidentityquality: Schema.optionalKey(Schema.Literal("source", "derived", "none")), data: Schema.Unknown, }).annotate({ identifier: "MapleCloudEvent" }) diff --git a/packages/eventing-core/src/registry.test.ts b/packages/eventing-core/src/registry.test.ts index d85a7161e..e8fe96d69 100644 --- a/packages/eventing-core/src/registry.test.ts +++ b/packages/eventing-core/src/registry.test.ts @@ -7,6 +7,7 @@ import { MAX_CLOUD_EVENT_BYTES, ProjectorRegistry, SignalSourceRegistry, + validateMapleCloudEvent, type NormalizedSignal, type SignalProjectionSpec, } from "./index" @@ -117,10 +118,22 @@ describe("CompiledProjectionRegistry", () => { type: "dev.maple.gitlab.issue.created.v1", subject: "project/example/issues/42", projectionrevision: 3, + sourceoccurrenceid: "event-123", + sourceidentityquality: "source", data: signal().data, }) }) + it("validates historical CloudEvents that predate source identity extensions", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const event = registry.evaluate(signal()).events[0]! + const { sourceoccurrenceid: _occurrence, sourceidentityquality: _quality, ...historical } = event + const validated = validateMapleCloudEvent(historical).event + expect(validated.id).toBe(event.id) + expect(validated.sourceoccurrenceid).toBeUndefined() + expect(validated.sourceidentityquality).toBeUndefined() + }) + it("runs every matching projection from one immutable registry snapshot", () => { const registry = CompiledProjectionRegistry.compile( [projection(), projection({ id: "gitlab-issue-created-audit" })], From 2492de067a787b79063592770692b84ba2a4b9e0 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 13 Aug 2026 21:30:43 -0400 Subject: [PATCH 09/12] fix(eventing): type source identity quality --- packages/eventing-core/schemas/cloud-event.v1.schema.json | 2 +- packages/eventing-core/src/model.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/eventing-core/schemas/cloud-event.v1.schema.json b/packages/eventing-core/schemas/cloud-event.v1.schema.json index 081aa10c7..323056917 100644 --- a/packages/eventing-core/schemas/cloud-event.v1.schema.json +++ b/packages/eventing-core/schemas/cloud-event.v1.schema.json @@ -155,7 +155,7 @@ }, "sourceidentityquality": { "type": "string", - "enum": ["source"] + "enum": ["source", "derived", "none"] }, "data": {} }, diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts index d74e57d83..5bdca978f 100644 --- a/packages/eventing-core/src/model.ts +++ b/packages/eventing-core/src/model.ts @@ -254,7 +254,7 @@ export const MapleCloudEventSchema = Schema.Struct({ projectorid: NonEmptyIdentifier, projectorversion: Schema.Int.check(Schema.isGreaterThan(0)), sourceoccurrenceid: Schema.optionalKey(NonEmptyIdentifier), - sourceidentityquality: Schema.optionalKey(Schema.Literal("source", "derived", "none")), + sourceidentityquality: Schema.optionalKey(Schema.Literals(["source", "derived", "none"])), data: Schema.Unknown, }).annotate({ identifier: "MapleCloudEvent" }) From 2ebd8d5a020c98ee70a18251372b892aafd6cb7a Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 13 Aug 2026 21:31:12 -0400 Subject: [PATCH 10/12] feat(cli): project complete GitLab event vocabulary --- .../src/server/eventing/gitlab-projectors.ts | 679 +++++++++++++++++- .../gitlab-projector-identities.v1.json | 46 ++ .../test/fixtures/gitlab-projectors.v1.json | 248 +++++++ apps/cli/test/gitlab-projectors.test.ts | 445 +++++++++++- 4 files changed, 1389 insertions(+), 29 deletions(-) create mode 100644 apps/cli/test/fixtures/gitlab-projector-identities.v1.json diff --git a/apps/cli/src/server/eventing/gitlab-projectors.ts b/apps/cli/src/server/eventing/gitlab-projectors.ts index 79af999da..39b3796dc 100644 --- a/apps/cli/src/server/eventing/gitlab-projectors.ts +++ b/apps/cli/src/server/eventing/gitlab-projectors.ts @@ -125,7 +125,48 @@ const PROJECT_LIFECYCLE_ACTIONS: Readonly> = } const TERMINAL_PIPELINE_STATUSES = new Set(["success", "failed", "canceled", "skipped"]) +const ISSUE_LIFECYCLE_ACTIONS = { + issue_open: "open", + issue_update: "update", + issue_close: "close", + issue_reopen: "reopen", +} as const +const MERGE_REQUEST_LIFECYCLE_ACTIONS = { + merge_request_open: "open", + merge_request_update: "update", + merge_request_close: "close", + merge_request_reopen: "reopen", + merge_request_merge: "merge", + merge_request_review: "review", +} as const +const DEPLOYMENT_STATUSES = { + deployment_running: "running", + deployment_success: "success", + deployment_failed: "failed", + deployment_canceled: "canceled", + deployment_blocked: "blocked", + deployment_manual: "manual", +} as const +const JOB_STATUSES = new Set([ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled", +]) +const RELEASE_ACTIONS = new Set(["create", "update", "delete"]) const MAX_PROJECTOR_TEXT_BYTES = 4 * 1024 +const MAX_COMMENT_EXCERPT_BYTES = 1024 +const MAX_CANONICAL_URL_BYTES = 2048 +const MAX_FAILED_JOBS = 20 +const MAX_ISSUE_LABELS = 50 +const MAX_ISSUE_LABEL_BYTES = 256 const projectorString = ( value: SignalScalar | undefined, @@ -144,6 +185,70 @@ const projectorString = ( return text } +const projectorToken = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + const token = projectorString(value, label, required) + if (token !== undefined && !/^[a-z][a-z0-9_]{0,63}$/.test(token)) + throw new Error(`GitLab projector ${label} must be a lowercase token`) + return token +} + +const projectorUrl = ( + value: SignalScalar | undefined, + label: string, + required = false, + allowFragment = false, +): string | undefined => { + const text = projectorString(value, label, required) + if (text === undefined) return undefined + if (Buffer.byteLength(text, "utf8") > MAX_CANONICAL_URL_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_CANONICAL_URL_BYTES} UTF-8 bytes`) + let parsed: URL + try { + parsed = new URL(text) + } catch { + throw new Error(`GitLab projector ${label} must be an absolute URL`) + } + if ((parsed.protocol !== "https:" && parsed.protocol !== "http:") || parsed.username || parsed.password) + throw new Error(`GitLab projector ${label} must be an HTTP(S) URL without credentials`) + if (parsed.search || (!allowFragment && parsed.hash)) + throw new Error(`GitLab projector ${label} must not contain a query or fragment`) + return text +} + +const projectorRevision = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + const revision = projectorString(value, label, required) + if (revision !== undefined && !/^[0-9a-f]{6,64}$/i.test(revision)) + throw new Error(`GitLab projector ${label} must be a hexadecimal revision`) + return revision +} + +const projectorBoolean = (value: SignalScalar | undefined, label: string): boolean | undefined => { + if (value === undefined) return undefined + if (value.type !== "boolean") throw new Error(`GitLab projector ${label} must be a boolean`) + return value.value +} + +const projectorExcerpt = (value: SignalScalar | undefined, label: string): string | undefined => { + const source = projectorString(value, label) + if (source === undefined) return undefined + const excerpt = source + .replace(/[\p{Cc}]/gu, " ") + .replace(/\s+/gu, " ") + .trim() + if (excerpt.length === 0) throw new Error(`GitLab projector ${label} must not be blank after sanitizing`) + if (Buffer.byteLength(excerpt, "utf8") > MAX_COMMENT_EXCERPT_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_COMMENT_EXCERPT_BYTES} UTF-8 bytes`) + return excerpt +} + const projectorInt64 = ( value: SignalScalar | undefined, label: string, @@ -193,13 +298,162 @@ const actor = (signal: NormalizedSignal) => { "gitlab.actor.id", ) const name = projectorString(field(signal, "attribute", "gitlab.actor.name"), "gitlab.actor.name") - if (name === undefined && id === undefined) return undefined + const username = projectorString( + field(signal, "attribute", "gitlab.actor.username"), + "gitlab.actor.username", + ) + if (name === undefined && username === undefined && id === undefined) return undefined return { ...(id === undefined ? {} : { id }), ...(name === undefined ? {} : { name }), + ...(username === undefined ? {} : { username }), + } +} + +const issue = (signal: NormalizedSignal) => { + const id = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.issue.id"), + "gitlab.issue.id", + ) + const iid = positiveProjectorInt64(field(signal, "attribute", "gitlab.issue.iid"), "gitlab.issue.iid") + const title = projectorString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") + const url = projectorUrl(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") + const state = projectorToken(field(signal, "attribute", "gitlab.issue.state"), "gitlab.issue.state") + const labels = issueLabels(signal) + return { + ...(id === undefined ? {} : { id }), + iid, + ...(title === undefined ? {} : { title }), + ...(url === undefined ? {} : { url }), + ...(state === undefined ? {} : { state }), + ...(labels === undefined ? {} : { labels }), + } +} + +const mergeRequest = (signal: NormalizedSignal) => { + const iid = positiveProjectorInt64( + field(signal, "attribute", "gitlab.merge_request.iid"), + "gitlab.merge_request.iid", + ) + const title = projectorString( + field(signal, "attribute", "gitlab.merge_request.title"), + "gitlab.merge_request.title", + ) + const url = projectorUrl( + field(signal, "attribute", "gitlab.merge_request.url"), + "gitlab.merge_request.url", + ) + const sourceBranch = projectorString( + field(signal, "attribute", "gitlab.merge_request.source_branch"), + "gitlab.merge_request.source_branch", + ) + const targetBranch = projectorString( + field(signal, "attribute", "gitlab.merge_request.target_branch"), + "gitlab.merge_request.target_branch", + ) + const commit = projectorRevision( + field(signal, "attribute", "gitlab.merge_request.commit"), + "gitlab.merge_request.commit", + ) + const reviewState = projectorToken( + field(signal, "attribute", "gitlab.merge_request.review_state"), + "gitlab.merge_request.review_state", + ) + return { + iid, + ...(title === undefined ? {} : { title }), + ...(url === undefined ? {} : { url }), + ...(sourceBranch === undefined ? {} : { sourceBranch }), + ...(targetBranch === undefined ? {} : { targetBranch }), + ...(commit === undefined ? {} : { commit }), + ...(reviewState === undefined ? {} : { reviewState }), + } +} + +const comment = (signal: NormalizedSignal, kind?: "comment" | "review") => { + const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.comment.id"), "gitlab.comment.id") + const excerpt = projectorExcerpt( + field(signal, "attribute", "gitlab.comment.excerpt"), + "gitlab.comment.excerpt", + ) + const url = projectorUrl( + field(signal, "attribute", "gitlab.comment.url"), + "gitlab.comment.url", + false, + true, + ) + const system = projectorBoolean( + field(signal, "attribute", "gitlab.comment.system"), + "gitlab.comment.system", + ) + return { + id, + ...(kind === undefined ? {} : { kind }), + ...(excerpt === undefined ? {} : { excerpt }), + ...(url === undefined ? {} : { url }), + ...(system === undefined ? {} : { system }), } } +const structuredAttribute = (signal: NormalizedSignal, key: string): unknown => { + if (!isRecord(signal.data) || !isRecord(signal.data.record) || !isRecord(signal.data.record.attributes)) + return undefined + return signal.data.record.attributes[key] +} + +const issueLabels = (signal: NormalizedSignal): readonly string[] | undefined => { + const value = structuredAttribute(signal, "gitlab.issue.labels") + if (value === undefined) return undefined + if (!Array.isArray(value)) throw new Error("GitLab projector gitlab.issue.labels must be an array") + if (value.length > MAX_ISSUE_LABELS) + throw new Error(`GitLab projector gitlab.issue.labels exceeds ${MAX_ISSUE_LABELS} labels`) + return value.map((candidate, index) => { + const label = `gitlab.issue.labels[${index}]` + if (typeof candidate !== "string") throw new Error(`GitLab projector ${label} must be a string`) + const normalized = candidate.trim() + if (normalized.length === 0) throw new Error(`GitLab projector ${label} must not be blank`) + if (Buffer.byteLength(normalized, "utf8") > MAX_ISSUE_LABEL_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_ISSUE_LABEL_BYTES} UTF-8 bytes`) + return normalized + }) +} + +const failedJobs = (signal: NormalizedSignal) => { + const value = structuredAttribute(signal, "gitlab.ci.pipeline.failed_jobs") + if (value === undefined) return undefined + if (!Array.isArray(value)) + throw new Error("GitLab projector gitlab.ci.pipeline.failed_jobs must be an array") + if (value.length > MAX_FAILED_JOBS) + throw new Error(`GitLab projector gitlab.ci.pipeline.failed_jobs exceeds ${MAX_FAILED_JOBS} jobs`) + return value.map((candidate, index) => { + const label = `gitlab.ci.pipeline.failed_jobs[${index}]` + if (!isRecord(candidate)) throw new Error(`GitLab projector ${label} must be an object`) + const allowed = new Set(["id", "name", "stage", "status", "url"]) + if (Object.keys(candidate).some((key) => !allowed.has(key))) + throw new Error(`GitLab projector ${label} contains an unknown field`) + const scalar = (key: string): SignalScalar | undefined => { + const item = candidate[key] + if (item === undefined) return undefined + if (typeof item !== "string") throw new Error(`GitLab projector ${label}.${key} must be a string`) + if (key === "id") return { type: "int64", value: item } + return { type: "string", value: item } + } + const id = positiveProjectorInt64(scalar("id"), `${label}.id`) + const name = projectorString(scalar("name"), `${label}.name`, true)! + const stage = projectorString(scalar("stage"), `${label}.stage`) + const status = projectorToken(scalar("status"), `${label}.status`, true)! + if (status !== "failed") throw new Error(`GitLab projector ${label}.status must be failed`) + const url = projectorUrl(scalar("url"), `${label}.url`) + return { + id, + name, + ...(stage === undefined ? {} : { stage }), + status, + ...(url === undefined ? {} : { url }), + } + }) +} + const project = (signal: NormalizedSignal) => { const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") const path = projectorString( @@ -263,6 +517,69 @@ export const gitlabProjectLifecycleProjector = { }, } as const +export const gitlabIssueLifecycleProjector = { + id: "gitlab.issue.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.issue.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const action = ISSUE_LIFECYCLE_ACTIONS[sourceEvent as keyof typeof ISSUE_LIFECYCLE_ACTIONS] + if (action === undefined) + throw new Error(`GitLab projector does not recognize issue event ${sourceEvent}`) + const projectData = project(signal) + const issueData = issue(signal) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/issues/${issueData.iid}`, + data: { + project: projectData, + issue: { ...issueData, action }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabIssueCommentProjector = { + id: "gitlab.issue.comment", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.comment.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-comment:v1", + decodeConfig: noProjectorConfig("gitlab.issue.comment"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + if (sourceEvent !== "issue_comment") + throw new Error(`GitLab projector does not recognize issue comment event ${sourceEvent}`) + const projectData = project(signal) + const issueData = issue(signal) + const commentData = comment(signal) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/issues/${issueData.iid}#note_${commentData.id}`, + data: { + project: projectData, + issue: issueData, + comment: commentData, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + export const gitlabMergeRequestLifecycleProjector = { id: "gitlab.merge-request.lifecycle", version: 1, @@ -272,39 +589,60 @@ export const gitlabMergeRequestLifecycleProjector = { decodeConfig: noProjectorConfig("gitlab.merge-request.lifecycle"), project: (signal: NormalizedSignal) => { const sourceEvent = eventName(signal) - const match = /^merge_request_([a-z][a-z0-9_]{0,63})$/.exec(sourceEvent) - if (!match) throw new Error(`GitLab projector does not recognize merge request event ${sourceEvent}`) + const action = + MERGE_REQUEST_LIFECYCLE_ACTIONS[sourceEvent as keyof typeof MERGE_REQUEST_LIFECYCLE_ACTIONS] + if (action === undefined) + throw new Error(`GitLab projector does not recognize merge request event ${sourceEvent}`) const projectData = project(signal) - const iid = positiveProjectorInt64( - field(signal, "attribute", "gitlab.merge_request.iid"), - "gitlab.merge_request.iid", - ) - const sourceBranch = projectorString( - field(signal, "attribute", "gitlab.merge_request.source_branch"), - "gitlab.merge_request.source_branch", - ) - const targetBranch = projectorString( - field(signal, "attribute", "gitlab.merge_request.target_branch"), - "gitlab.merge_request.target_branch", - ) - const commit = projectorString( - field(signal, "attribute", "gitlab.merge_request.commit"), - "gitlab.merge_request.commit", - ) + const mergeRequestData = mergeRequest(signal) + if (action === "review" && mergeRequestData.reviewState === undefined) + throw new Error("GitLab projector is missing gitlab.merge_request.review_state") const eventActor = actor(signal) const eventResult = result(signal) const eventServiceName = serviceName(signal) return { - subject: `${projectData.path}/-/merge_requests/${iid}`, + subject: `${projectData.path}/-/merge_requests/${mergeRequestData.iid}`, data: { project: projectData, - mergeRequest: { - iid, - action: match[1]!, - ...(sourceBranch === undefined ? {} : { sourceBranch }), - ...(targetBranch === undefined ? {} : { targetBranch }), - ...(commit === undefined ? {} : { commit }), - }, + mergeRequest: { ...mergeRequestData, action }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabMergeRequestCommentProjector = { + id: "gitlab.merge-request.comment", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.merge-request.comment.v1", + dataSchema: "urn:maple:event-schema:gitlab-merge-request-comment:v1", + decodeConfig: noProjectorConfig("gitlab.merge-request.comment"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const kind = + sourceEvent === "merge_request_comment" + ? "comment" + : sourceEvent === "merge_request_review_comment" + ? "review" + : undefined + if (kind === undefined) + throw new Error(`GitLab projector does not recognize merge request comment event ${sourceEvent}`) + const projectData = project(signal) + const mergeRequestData = mergeRequest(signal) + const commentData = comment(signal, kind) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/merge_requests/${mergeRequestData.iid}#note_${commentData.id}`, + data: { + project: projectData, + mergeRequest: mergeRequestData, + comment: commentData, sourceEvent, ...(eventActor === undefined ? {} : { actor: eventActor }), ...(eventResult === undefined ? {} : { result: eventResult }), @@ -334,6 +672,10 @@ export const gitlabPipelineCompletedProjector = { field(signal, "attribute", "gitlab.ci.pipeline.iid"), "gitlab.ci.pipeline.iid", ) + const mergeRequestIid = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.merge_request_iid"), + "gitlab.ci.pipeline.merge_request_iid", + ) const status = projectorString( field(signal, "attribute", "gitlab.ci.pipeline.status"), "gitlab.ci.pipeline.status", @@ -353,6 +695,10 @@ export const gitlabPipelineCompletedProjector = { field(signal, "attribute", "gitlab.ci.pipeline.detailed_status"), "gitlab.ci.pipeline.detailed_status", ) + const url = projectorUrl( + field(signal, "attribute", "gitlab.ci.pipeline.url"), + "gitlab.ci.pipeline.url", + ) const ref = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref") const sha = projectorString( field(signal, "attribute", "vcs.ref.head.revision"), @@ -370,6 +716,29 @@ export const gitlabPipelineCompletedProjector = { field(signal, "attribute", "gitlab.ci.pipeline.stage_count"), "gitlab.ci.pipeline.stage_count", ) + const failedJobCount = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.failed_job_count"), + "gitlab.ci.pipeline.failed_job_count", + ) + const failedJobsTruncated = projectorBoolean( + field(signal, "attribute", "gitlab.ci.pipeline.failed_jobs_truncated"), + "gitlab.ci.pipeline.failed_jobs_truncated", + ) + const pipelineFailedJobs = failedJobs(signal) + if ((failedJobCount === undefined) !== (failedJobsTruncated === undefined)) + throw new Error("GitLab projector failed job count and truncation flag must be supplied together") + if (pipelineFailedJobs !== undefined) { + if (status !== "failed") + throw new Error("GitLab projector failed jobs are only valid for a failed pipeline") + if (failedJobCount === undefined) + throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_job_count") + if (failedJobsTruncated === undefined) + throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_jobs_truncated") + if (BigInt(failedJobCount) < BigInt(pipelineFailedJobs.length)) + throw new Error("GitLab projector failed job count is smaller than the summary list") + if (failedJobsTruncated !== BigInt(failedJobCount) > BigInt(pipelineFailedJobs.length)) + throw new Error("GitLab projector failed job truncation flag is inconsistent") + } const eventActor = actor(signal) const eventServiceName = serviceName(signal) const eventResult = result(signal) @@ -380,15 +749,264 @@ export const gitlabPipelineCompletedProjector = { pipeline: { id, ...(iid === undefined ? {} : { iid }), + ...(mergeRequestIid === undefined ? {} : { mergeRequestIid }), status, ...(name === undefined ? {} : { name }), ...(pipelineSource === undefined ? {} : { source: pipelineSource }), ...(detailedStatus === undefined ? {} : { detailedStatus }), + ...(url === undefined ? {} : { url }), ...(ref === undefined ? {} : { ref }), ...(sha === undefined ? {} : { sha }), ...(durationMs === undefined ? {} : { durationMs }), ...(queuedDurationMs === undefined ? {} : { queuedDurationMs }), ...(stageCount === undefined ? {} : { stageCount }), + ...(failedJobCount === undefined ? {} : { failedJobCount }), + ...(failedJobsTruncated === undefined ? {} : { failedJobsTruncated }), + ...(pipelineFailedJobs === undefined ? {} : { failedJobs: pipelineFailedJobs }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabDeploymentLifecycleProjector = { + id: "gitlab.deployment.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.deployment.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.deployment.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const expectedStatus = DEPLOYMENT_STATUSES[sourceEvent as keyof typeof DEPLOYMENT_STATUSES] + if (expectedStatus === undefined) + throw new Error(`GitLab projector does not recognize deployment event ${sourceEvent}`) + const projectData = project(signal) + const id = positiveProjectorInt64( + field(signal, "attribute", "gitlab.deployment.id"), + "gitlab.deployment.id", + ) + const environment = projectorString( + field(signal, "attribute", "gitlab.deployment.environment"), + "gitlab.deployment.environment", + true, + )! + const status = projectorToken( + field(signal, "attribute", "gitlab.deployment.status"), + "gitlab.deployment.status", + true, + )! + if (status !== expectedStatus) + throw new Error(`GitLab projector deployment status ${status} conflicts with ${sourceEvent}`) + const revision = projectorRevision( + field(signal, "attribute", "gitlab.deployment.revision"), + "gitlab.deployment.revision", + ) + const url = projectorUrl(field(signal, "attribute", "gitlab.deployment.url"), "gitlab.deployment.url") + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/deployments/${id}`, + data: { + project: projectData, + deployment: { + id, + environment, + status, + ...(revision === undefined ? {} : { revision }), + ...(url === undefined ? {} : { url }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabJobLifecycleProjector = { + id: "gitlab.job.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.job.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-job-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.job.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const match = /^ci_job_([a-z][a-z0-9_]{0,63})$/.exec(sourceEvent) + const action = match?.[1] + if (action === undefined || !JOB_STATUSES.has(action)) + throw new Error(`GitLab projector does not recognize job event ${sourceEvent}`) + const projectData = project(signal) + const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.ci.job.id"), "gitlab.ci.job.id") + const name = projectorString( + field(signal, "attribute", "gitlab.ci.job.name"), + "gitlab.ci.job.name", + true, + )! + const stage = projectorString( + field(signal, "attribute", "gitlab.ci.job.stage"), + "gitlab.ci.job.stage", + ) + const status = projectorToken( + field(signal, "attribute", "gitlab.ci.job.status"), + "gitlab.ci.job.status", + true, + )! + if (status !== action) + throw new Error(`GitLab projector job status ${status} conflicts with ${sourceEvent}`) + const url = projectorUrl(field(signal, "attribute", "gitlab.ci.job.url"), "gitlab.ci.job.url") + const pipelineId = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.id"), + "gitlab.ci.pipeline.id", + ) + const ref = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref") + const revision = projectorRevision( + field(signal, "attribute", "vcs.ref.head.revision"), + "vcs.ref.head.revision", + ) + const durationMs = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.job.duration_ms"), + "gitlab.ci.job.duration_ms", + ) + const allowFailure = projectorBoolean( + field(signal, "attribute", "gitlab.ci.job.allow_failure"), + "gitlab.ci.job.allow_failure", + ) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/jobs/${id}`, + data: { + project: projectData, + job: { + id, + name, + status, + ...(stage === undefined ? {} : { stage }), + ...(url === undefined ? {} : { url }), + ...(pipelineId === undefined ? {} : { pipelineId }), + ...(ref === undefined ? {} : { ref }), + ...(revision === undefined ? {} : { revision }), + ...(durationMs === undefined ? {} : { durationMs }), + ...(allowFailure === undefined ? {} : { allowFailure }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +const REF_EVENTS = { + branch_create: { type: "branch", action: "create" }, + branch_update: { type: "branch", action: "update" }, + branch_delete: { type: "branch", action: "delete" }, + tag_create: { type: "tag", action: "create" }, + tag_update: { type: "tag", action: "update" }, + tag_delete: { type: "tag", action: "delete" }, +} as const + +export const gitlabRefLifecycleProjector = { + id: "gitlab.ref.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.ref.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-ref-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.ref.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const lifecycle = REF_EVENTS[sourceEvent as keyof typeof REF_EVENTS] + if (lifecycle === undefined) + throw new Error(`GitLab projector does not recognize ref event ${sourceEvent}`) + const projectData = project(signal) + const name = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref", true)! + const before = projectorRevision( + field(signal, "attribute", "vcs.ref.base.revision"), + "vcs.ref.base.revision", + true, + )! + const after = projectorRevision( + field(signal, "attribute", "vcs.ref.head.revision"), + "vcs.ref.head.revision", + true, + )! + const url = projectorUrl(field(signal, "attribute", "gitlab.ref.url"), "gitlab.ref.url") + const commitCount = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.push.commit_count"), + "gitlab.push.commit_count", + ) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/refs/${name}`, + data: { + project: projectData, + ref: { + ...lifecycle, + name, + before, + after, + ...(url === undefined ? {} : { url }), + ...(commitCount === undefined ? {} : { commitCount }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabReleaseLifecycleProjector = { + id: "gitlab.release.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.release.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-release-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.release.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const match = /^release_([a-z][a-z0-9_]{0,63})$/.exec(sourceEvent) + const action = match?.[1] + if (action === undefined || !RELEASE_ACTIONS.has(action)) + throw new Error(`GitLab projector does not recognize release event ${sourceEvent}`) + const projectData = project(signal) + const id = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.release.id"), + "gitlab.release.id", + ) + const tag = projectorString( + field(signal, "attribute", "gitlab.release.tag"), + "gitlab.release.tag", + true, + )! + const name = projectorString(field(signal, "attribute", "gitlab.release.name"), "gitlab.release.name") + const url = projectorUrl(field(signal, "attribute", "gitlab.release.url"), "gitlab.release.url") + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/releases/${tag}`, + data: { + project: projectData, + release: { + ...(id === undefined ? {} : { id }), + tag, + action, + ...(name === undefined ? {} : { name }), + ...(url === undefined ? {} : { url }), }, sourceEvent, ...(eventActor === undefined ? {} : { actor: eventActor }), @@ -403,5 +1021,12 @@ export const registerGitLabProjectors = (registry: ProjectorRegistry): Projector registry .register(gitlabIssueCreatedProjector) .register(gitlabProjectLifecycleProjector) + .register(gitlabIssueLifecycleProjector) + .register(gitlabIssueCommentProjector) .register(gitlabMergeRequestLifecycleProjector) + .register(gitlabMergeRequestCommentProjector) .register(gitlabPipelineCompletedProjector) + .register(gitlabDeploymentLifecycleProjector) + .register(gitlabJobLifecycleProjector) + .register(gitlabRefLifecycleProjector) + .register(gitlabReleaseLifecycleProjector) diff --git a/apps/cli/test/fixtures/gitlab-projector-identities.v1.json b/apps/cli/test/fixtures/gitlab-projector-identities.v1.json new file mode 100644 index 000000000..2022db7ea --- /dev/null +++ b/apps/cli/test/fixtures/gitlab-projector-identities.v1.json @@ -0,0 +1,46 @@ +[ + { + "occurrenceId": "project-event-42", + "projectionId": "gitlab-project-renamed" + }, + { + "occurrenceId": "mr-event-7", + "projectionId": "gitlab-mr-opened" + }, + { + "occurrenceId": "pipeline-event-900", + "projectionId": "gitlab-pipeline-completed" + }, + { + "occurrenceId": "delivery-issue:issue_close", + "projectionId": "gitlab-issue_close" + }, + { + "occurrenceId": "delivery-comment:0", + "projectionId": "gitlab-issue-comment" + }, + { + "occurrenceId": "delivery-mr-comment:merge_request_review_comment", + "projectionId": "gitlab-merge_request_review_comment" + }, + { + "occurrenceId": "delivery-pipeline:0", + "projectionId": "gitlab-pipeline-enriched" + }, + { + "occurrenceId": "delivery-deployment:deployment_failed", + "projectionId": "gitlab-deployment_failed" + }, + { + "occurrenceId": "delivery-job:failed", + "projectionId": "gitlab-ci_job_failed" + }, + { + "occurrenceId": "delivery-ref:tag_create", + "projectionId": "gitlab-tag_create" + }, + { + "occurrenceId": "delivery-release:create", + "projectionId": "gitlab-release_create" + } +] diff --git a/apps/cli/test/fixtures/gitlab-projectors.v1.json b/apps/cli/test/fixtures/gitlab-projectors.v1.json index 003a013ee..02c223964 100644 --- a/apps/cli/test/fixtures/gitlab-projectors.v1.json +++ b/apps/cli/test/fixtures/gitlab-projectors.v1.json @@ -13,6 +13,8 @@ "projectionrevision": 1, "projectorid": "gitlab.project.lifecycle", "projectorversion": 1, + "sourceoccurrenceid": "project-event-42", + "sourceidentityquality": "source", "data": { "project": { "id": "42", "path": "rdev/maple", "oldPath": "rdev/old-maple" }, "action": "renamed", @@ -36,6 +38,8 @@ "projectionrevision": 1, "projectorid": "gitlab.merge-request.lifecycle", "projectorversion": 1, + "sourceoccurrenceid": "mr-event-7", + "sourceidentityquality": "source", "data": { "project": { "id": "42", "path": "rdev/maple" }, "mergeRequest": { @@ -65,6 +69,8 @@ "projectionrevision": 1, "projectorid": "gitlab.pipeline.completed", "projectorversion": 1, + "sourceoccurrenceid": "pipeline-event-900", + "sourceidentityquality": "source", "data": { "project": { "id": "42", "path": "rdev/maple" }, "pipeline": { @@ -85,5 +91,247 @@ "result": "failure", "serviceName": "gitlab-repository-events" } + }, + { + "specversion": "1.0", + "id": "sha256:880765fe8243154db7a7c1902ff328ca360da3acda28c9081539e35aea16798b", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.issue.lifecycle.v1", + "subject": "rdev/maple/-/issues/7", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-issue-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-issue_close", + "projectionrevision": 1, + "projectorid": "gitlab.issue.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-issue:issue_close", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "issue": { + "id": "70", + "iid": "7", + "title": "Typed events", + "url": "https://gitlab.internal/rdev/maple/-/issues/7", + "labels": ["agent-ready", "backend"], + "action": "close" + }, + "sourceEvent": "issue_close", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:b3e27095a6aa053a50b2842109bb0a48ec5140db48ae6af1f13352ae14dbe515", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.issue.comment.v1", + "subject": "rdev/maple/-/issues/7#note_81", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-issue-comment:v1", + "tenantid": "local", + "projectionid": "gitlab-issue-comment", + "projectionrevision": 1, + "projectorid": "gitlab.issue.comment", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-comment:0", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "issue": { "iid": "7", "labels": ["agent-ready", "backend"] }, + "comment": { + "id": "81", + "excerpt": "hello Maple", + "url": "https://gitlab.internal/rdev/maple/-/issues/7#note_81" + }, + "sourceEvent": "issue_comment", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:6e2f7dd052deee38d40f3f0dc37a3df0f42e753c322d65be0eaed74ced497016", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.merge-request.comment.v1", + "subject": "rdev/maple/-/merge_requests/7#note_82", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-merge-request-comment:v1", + "tenantid": "local", + "projectionid": "gitlab-merge_request_review_comment", + "projectionrevision": 1, + "projectorid": "gitlab.merge-request.comment", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-mr-comment:merge_request_review_comment", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "mergeRequest": { "iid": "7" }, + "comment": { "id": "82", "kind": "review", "excerpt": "Please add a fixture" }, + "sourceEvent": "merge_request_review_comment", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:7c9304b99e00f9d8be2e758bf8b98c78387f22be23f415810ec3d44fed3d4c89", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.pipeline.completed.v1", + "subject": "rdev/maple/-/pipelines/900", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-pipeline-completed:v1", + "tenantid": "local", + "projectionid": "gitlab-pipeline-enriched", + "projectionrevision": 1, + "projectorid": "gitlab.pipeline.completed", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-pipeline:0", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "pipeline": { + "id": "900", + "mergeRequestIid": "7", + "status": "failed", + "url": "https://gitlab.internal/rdev/maple/-/pipelines/900", + "failedJobCount": "3", + "failedJobsTruncated": true, + "failedJobs": [ + { + "id": "901", + "name": "unit", + "stage": "test", + "status": "failed", + "url": "https://gitlab.internal/rdev/maple/-/jobs/901" + }, + { "id": "902", "name": "integration", "stage": "test", "status": "failed" } + ] + }, + "sourceEvent": "ci_pipeline_completed", + "actor": { "id": "9", "name": "rdev" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:b7b29d2a12f061dc3e8b6c3bcb550302783b834541b6a8db7b104b4d14f41464", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.deployment.lifecycle.v1", + "subject": "rdev/maple/-/deployments/501", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-deployment_failed", + "projectionrevision": 1, + "projectorid": "gitlab.deployment.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-deployment:deployment_failed", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "deployment": { + "id": "501", + "environment": "production", + "status": "failed", + "revision": "a8123fa", + "url": "https://gitlab.internal/rdev/maple/-/deployments/501" + }, + "sourceEvent": "deployment_failed", + "actor": { "id": "9", "name": "rdev" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:88af5c5523b40f64488a3fa8134139813974973ca826197006d414802fad03fe", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.job.lifecycle.v1", + "subject": "rdev/maple/-/jobs/901", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-job-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-ci_job_failed", + "projectionrevision": 1, + "projectorid": "gitlab.job.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-job:failed", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "job": { "id": "901", "name": "unit", "status": "failed" }, + "sourceEvent": "ci_job_failed", + "actor": { "id": "9", "name": "rdev" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:30adaaa10eb7d5dace8a2440e6b110f06d77948fe605d3b6cb274d8b2433b804", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.ref.lifecycle.v1", + "subject": "rdev/maple/-/refs/refs/tags/v1.0.0", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-ref-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-tag_create", + "projectionrevision": 1, + "projectorid": "gitlab.ref.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-ref:tag_create", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "ref": { + "type": "tag", + "action": "create", + "name": "refs/tags/v1.0.0", + "before": "0000000000000000000000000000000000000000", + "after": "a8123faa8123faa8123faa8123faa8123faa8123" + }, + "sourceEvent": "tag_create", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:d784c81bef9413f2132218d086ab15f81ec49ef9ec833becb622f92fac58f617", + "source": "https://gitlab.internal", + "type": "dev.maple.gitlab.release.lifecycle.v1", + "subject": "rdev/maple/-/releases/v1.0.0", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-release-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-release_create", + "projectionrevision": 1, + "projectorid": "gitlab.release.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-release:create", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "rdev/maple" }, + "release": { "tag": "v1.0.0", "action": "create", "name": "Maple 1.0" }, + "sourceEvent": "release_create", + "actor": { "id": "9", "name": "rdev" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } } ] diff --git a/apps/cli/test/gitlab-projectors.test.ts b/apps/cli/test/gitlab-projectors.test.ts index 6707b8eaf..3a206615c 100644 --- a/apps/cli/test/gitlab-projectors.test.ts +++ b/apps/cli/test/gitlab-projectors.test.ts @@ -4,12 +4,17 @@ import { CompiledProjectionRegistry, ProjectorRegistry, SignalSourceRegistry, + makeEventId, validateMapleCloudEvent, type SignalPredicate, type SignalProjectionSpec, } from "@maple/eventing-core" import samples from "./fixtures/gitlab-projectors.v1.json" +import identities from "./fixtures/gitlab-projector-identities.v1.json" import { + gitlabDeploymentLifecycleProjector, + gitlabIssueCommentProjector, + gitlabJobLifecycleProjector, gitlabMergeRequestLifecycleProjector, gitlabPipelineCompletedProjector, gitlabProjectLifecycleProjector, @@ -19,6 +24,38 @@ import { OTLP_LOG_ADAPTER, normalizeOtlpLogs } from "../src/server/eventing/otlp const stringAttr = (key: string, value: string) => ({ key, value: { stringValue: value } }) const intAttr = (key: string, value: string) => ({ key, value: { intValue: value } }) +const boolAttr = (key: string, value: boolean) => ({ key, value: { boolValue: value } }) +const stringArrayAttr = (key: string, values: readonly string[]) => ({ + key, + value: { arrayValue: { values: values.map((value) => ({ stringValue: value })) } }, +}) +const failedJobsAttr = ( + jobs: ReadonlyArray<{ + readonly id: string + readonly name: string + readonly stage?: string + readonly url?: string + }>, +) => ({ + key: "gitlab.ci.pipeline.failed_jobs", + value: { + arrayValue: { + values: jobs.map((job) => ({ + kvlistValue: { + values: [ + { key: "id", value: { intValue: job.id } }, + { key: "name", value: { stringValue: job.name } }, + ...(job.stage === undefined + ? [] + : [{ key: "stage", value: { stringValue: job.stage } }]), + { key: "status", value: { stringValue: "failed" } }, + ...(job.url === undefined ? [] : [{ key: "url", value: { stringValue: job.url } }]), + ], + }, + })), + }, + }, +}) const gitlabEvent = (eventName: string, eventId: string, extra: readonly unknown[] = []) => ({ resourceLogs: [ @@ -232,6 +269,281 @@ describe("GitLab event projectors", () => { strictEqual(result.events[0]?.subject, "rdev/maple/-/pipelines/900") }) + it("projects every issue lifecycle action and sanitized issue comments", () => { + for (const [sourceEvent, action] of Object.entries({ + issue_open: "open", + issue_update: "update", + issue_close: "close", + issue_reopen: "reopen", + })) { + const result = evaluate( + "gitlab.issue.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-issue:${sourceEvent}`, [ + intAttr("gitlab.issue.id", "70"), + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.title", "Typed events"), + stringAttr("gitlab.issue.url", "https://gitlab.internal/rdev/maple/-/issues/7"), + stringArrayAttr("gitlab.issue.labels", ["agent-ready", "backend"]), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual((result.events[0]?.data as { issue: { action: string } }).issue.action, action) + } + + const commentResult = evaluate( + "gitlab.issue.comment", + "gitlab-issue-comment", + "issue_comment", + normalizedEvent( + gitlabEvent("issue_comment", "delivery-comment:0", [ + intAttr("gitlab.issue.iid", "7"), + intAttr("gitlab.comment.id", "81"), + stringArrayAttr("gitlab.issue.labels", ["agent-ready", "backend"]), + stringAttr("gitlab.comment.excerpt", " hello\u0000\n\tMaple "), + stringAttr("gitlab.comment.url", "https://gitlab.internal/rdev/maple/-/issues/7#note_81"), + ]), + ), + ) + strictEqual(commentResult.failures.length, 0) + deepStrictEqual(commentResult.events[0]?.data, { + project: { id: "42", path: "rdev/maple" }, + issue: { iid: "7", labels: ["agent-ready", "backend"] }, + comment: { + id: "81", + excerpt: "hello Maple", + url: "https://gitlab.internal/rdev/maple/-/issues/7#note_81", + }, + sourceEvent: "issue_comment", + actor: { id: "9", name: "rdev" }, + result: "success", + serviceName: "gitlab-repository-events", + }) + }) + + it("uses an explicit MR lifecycle vocabulary and distinguishes review comments", () => { + for (const [sourceEvent, action] of Object.entries({ + merge_request_open: "open", + merge_request_update: "update", + merge_request_close: "close", + merge_request_reopen: "reopen", + merge_request_merge: "merge", + merge_request_review: "review", + })) { + const extras = [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.internal/rdev/maple/-/merge_requests/7", + ), + ...(sourceEvent === "merge_request_review" + ? [stringAttr("gitlab.merge_request.review_state", "approved")] + : []), + ] + const result = evaluate( + "gitlab.merge-request.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent(gitlabEvent(sourceEvent, `delivery-mr:${sourceEvent}`, extras)), + ) + strictEqual(result.failures.length, 0) + strictEqual( + (result.events[0]?.data as { mergeRequest: { action: string } }).mergeRequest.action, + action, + ) + } + + for (const [sourceEvent, kind] of [ + ["merge_request_comment", "comment"], + ["merge_request_review_comment", "review"], + ] as const) { + const result = evaluate( + "gitlab.merge-request.comment", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-mr-comment:${sourceEvent}`, [ + intAttr("gitlab.merge_request.iid", "7"), + intAttr("gitlab.comment.id", "82"), + stringAttr("gitlab.comment.excerpt", "Please add a fixture"), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual((result.events[0]?.data as { comment: { kind: string } }).comment.kind, kind) + } + }) + + it("projects bounded failed-job summaries with canonical pipeline metadata", () => { + const result = evaluate( + "gitlab.pipeline.completed", + "gitlab-pipeline-enriched", + "ci_pipeline_completed", + normalizedEvent( + gitlabEvent("ci_pipeline_completed", "delivery-pipeline:0", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + intAttr("gitlab.ci.pipeline.merge_request_iid", "7"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.internal/rdev/maple/-/pipelines/900", + ), + intAttr("gitlab.ci.pipeline.failed_job_count", "3"), + boolAttr("gitlab.ci.pipeline.failed_jobs_truncated", true), + failedJobsAttr([ + { + id: "901", + name: "unit", + stage: "test", + url: "https://gitlab.internal/rdev/maple/-/jobs/901", + }, + { id: "902", name: "integration", stage: "test" }, + ]), + ]), + ), + ) + strictEqual(result.failures.length, 0) + deepStrictEqual( + (result.events[0]?.data as { pipeline: { failedJobCount: string; failedJobsTruncated: boolean } }) + .pipeline, + { + id: "900", + mergeRequestIid: "7", + status: "failed", + url: "https://gitlab.internal/rdev/maple/-/pipelines/900", + failedJobCount: "3", + failedJobsTruncated: true, + failedJobs: [ + { + id: "901", + name: "unit", + stage: "test", + status: "failed", + url: "https://gitlab.internal/rdev/maple/-/jobs/901", + }, + { id: "902", name: "integration", stage: "test", status: "failed" }, + ], + }, + ) + }) + + it("projects deployment and job lifecycle contracts with status agreement", () => { + for (const [sourceEvent, status] of Object.entries({ + deployment_running: "running", + deployment_success: "success", + deployment_failed: "failed", + deployment_canceled: "canceled", + deployment_blocked: "blocked", + deployment_manual: "manual", + })) { + const result = evaluate( + "gitlab.deployment.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-deployment:${sourceEvent}`, [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "production"), + stringAttr("gitlab.deployment.status", status), + stringAttr("gitlab.deployment.revision", "a8123fa"), + stringAttr( + "gitlab.deployment.url", + "https://gitlab.internal/rdev/maple/-/deployments/501", + ), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual( + (result.events[0]?.data as { deployment: { status: string } }).deployment.status, + status, + ) + } + + for (const status of [ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled", + ]) { + const sourceEvent = `ci_job_${status}` + const result = evaluate( + "gitlab.job.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-job:${status}`, [ + intAttr("gitlab.ci.job.id", "901"), + stringAttr("gitlab.ci.job.name", "unit"), + stringAttr("gitlab.ci.job.status", status), + ]), + ), + ) + strictEqual(result.failures.length, 0) + } + }) + + it("projects normalized branch/tag transitions and release lifecycle without descriptions", () => { + for (const [sourceEvent, type, action] of [ + ["branch_create", "branch", "create"], + ["branch_update", "branch", "update"], + ["branch_delete", "branch", "delete"], + ["tag_create", "tag", "create"], + ["tag_update", "tag", "update"], + ["tag_delete", "tag", "delete"], + ] as const) { + const result = evaluate( + "gitlab.ref.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-ref:${sourceEvent}`, [ + stringAttr("vcs.ref", type === "branch" ? "refs/heads/main" : "refs/tags/v1.0.0"), + stringAttr("vcs.ref.base.revision", "0000000000000000000000000000000000000000"), + stringAttr("vcs.ref.head.revision", "a8123faa8123faa8123faa8123faa8123faa8123"), + ]), + ), + ) + strictEqual(result.failures.length, 0) + deepStrictEqual((result.events[0]?.data as { ref: { type: string; action: string } }).ref, { + type, + action, + name: type === "branch" ? "refs/heads/main" : "refs/tags/v1.0.0", + before: "0000000000000000000000000000000000000000", + after: "a8123faa8123faa8123faa8123faa8123faa8123", + }) + } + + for (const action of ["create", "update", "delete"]) { + const sourceEvent = `release_${action}` + const result = evaluate( + "gitlab.release.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-release:${action}`, [ + stringAttr("gitlab.release.tag", "v1.0.0"), + stringAttr("gitlab.release.name", "Maple 1.0"), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual((result.events[0]?.data as { release: { action: string } }).release.action, action) + strictEqual("description" in (result.events[0]?.data as { release: object }).release, false) + } + }) + it("keeps source identity and the CloudEvent ID deterministic across retries", () => { const first = normalizedEvent(gitlabEvent("project_create", "project-event-42")) const retry = normalizedEvent(gitlabEvent("project_create", "project-event-42")) @@ -302,14 +614,143 @@ describe("GitLab event projectors", () => { () => gitlabMergeRequestLifecycleProjector.project(unknownAction), /GitLab projector does not recognize merge request event/, ) + const unknownBoundedAction = normalizedEvent( + gitlabEvent("merge_request_deploy", "mr-invalid-bounded", [ + intAttr("gitlab.merge_request.iid", "7"), + ]), + ) + throws( + () => gitlabMergeRequestLifecycleProjector.project(unknownBoundedAction), + /GitLab projector does not recognize merge request event/, + ) throws( () => gitlabProjectLifecycleProjector.decodeConfig({ includeBody: true }), /gitlab\.project\.lifecycle projector config contains unknown fields/, ) }) + it("fails closed on inconsistent statuses, unsafe URLs, and collection bounds", () => { + const deploymentMismatch = normalizedEvent( + gitlabEvent("deployment_success", "deployment-mismatch", [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "production"), + stringAttr("gitlab.deployment.status", "failed"), + ]), + ) + throws( + () => gitlabDeploymentLifecycleProjector.project(deploymentMismatch), + /status failed conflicts with deployment_success/, + ) + + const jobMismatch = normalizedEvent( + gitlabEvent("ci_job_success", "job-mismatch", [ + intAttr("gitlab.ci.job.id", "901"), + stringAttr("gitlab.ci.job.name", "unit"), + stringAttr("gitlab.ci.job.status", "failed"), + ]), + ) + throws(() => gitlabJobLifecycleProjector.project(jobMismatch), /status failed conflicts/) + + const unsafeUrl = normalizedEvent( + gitlabEvent("issue_comment", "comment-query", [ + intAttr("gitlab.issue.iid", "7"), + intAttr("gitlab.comment.id", "81"), + stringAttr("gitlab.comment.url", "https://gitlab.internal/note?private=value"), + ]), + ) + throws(() => gitlabIssueCommentProjector.project(unsafeUrl), /must not contain a query or fragment/) + + const tooManyJobs = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-too-many-jobs", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + failedJobsAttr( + Array.from({ length: 21 }, (_, index) => ({ + id: String(index + 1), + name: `job-${index}`, + })), + ), + ]), + ) + throws(() => gitlabPipelineCompletedProjector.project(tooManyJobs), /exceeds 20 jobs/) + + const oversizedExcerpt = normalizedEvent( + gitlabEvent("issue_comment", "comment-too-long", [ + intAttr("gitlab.issue.iid", "7"), + intAttr("gitlab.comment.id", "81"), + stringAttr("gitlab.comment.excerpt", "x".repeat(1025)), + ]), + ) + throws(() => gitlabIssueCommentProjector.project(oversizedExcerpt), /exceeds 1024 UTF-8 bytes/) + + const tooManyLabels = normalizedEvent( + gitlabEvent("issue_open", "issue-too-many-labels", [ + intAttr("gitlab.issue.iid", "7"), + stringArrayAttr( + "gitlab.issue.labels", + Array.from({ length: 51 }, (_, index) => `label-${index}`), + ), + ]), + ) + const labelsResult = evaluate( + "gitlab.issue.lifecycle", + "gitlab-issue-labels-bounded", + "issue_open", + tooManyLabels, + ) + strictEqual( + labelsResult.failures[0]?.message, + "GitLab projector gitlab.issue.labels exceeds 50 labels", + ) + }) + + it("preserves indexed producer occurrence IDs as distinct deterministic CloudEvent identities", () => { + const request = gitlabEvent("project_update", "delivery-uuid:0") + const second = structuredClone(request.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!) + second.attributes = second.attributes.map((entry) => + entry.key === "event.id" || entry.key === "gitlab.event.id" + ? stringAttr(entry.key, "delivery-uuid:1") + : entry, + ) + request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(second) + const signals = normalizeOtlpLogs(request, "2026-08-07T20:00:00Z") + deepStrictEqual( + signals.map(({ occurrenceId }) => occurrenceId), + ["delivery-uuid:0", "delivery-uuid:1"], + ) + const ids = signals.map( + (signal) => + evaluate("gitlab.project.lifecycle", "gitlab-project-updated", "project_update", signal) + .events[0]?.id, + ) + strictEqual(new Set(ids).size, 2) + deepStrictEqual( + ids, + signals.map( + (signal) => + evaluate("gitlab.project.lifecycle", "gitlab-project-updated", "project_update", signal) + .events[0]?.id, + ), + ) + }) + it("keeps the checked-in sample CloudEvents envelope-valid", () => { - strictEqual(samples.length, 3) - for (const sample of samples) strictEqual(validateMapleCloudEvent(sample).event.id, sample.id) + strictEqual(samples.length, 11) + strictEqual(identities.length, samples.length) + for (const [index, sample] of samples.entries()) { + strictEqual(validateMapleCloudEvent(sample).event.id, sample.id) + const identity = identities[index]! + strictEqual( + makeEventId({ + tenantId: "local", + sourceKind: "otel.log", + source: "https://gitlab.internal", + occurrenceId: identity.occurrenceId, + projectionId: identity.projectionId, + projectionRevision: 1, + }), + sample.id, + ) + } }) }) From 2cb2ceea5a9786b60f2731c159fa65654b337d77 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 13 Aug 2026 21:31:23 -0400 Subject: [PATCH 11/12] docs(eventing): define GitLab v1 contracts --- docs/gitlab-event-projectors.md | 400 +++++++++++++++++++++++--------- 1 file changed, 286 insertions(+), 114 deletions(-) diff --git a/docs/gitlab-event-projectors.md b/docs/gitlab-event-projectors.md index c90b9dce9..9a8b5c99a 100644 --- a/docs/gitlab-event-projectors.md +++ b/docs/gitlab-event-projectors.md @@ -2,167 +2,339 @@ Status: version-1 producer contracts for the Maple Local eventing outbox. -This document describes the projectors registered by the Local runtime. They -normalize the already bounded OTLP fields emitted by the GitLab repository-event -receiver. They do not call GitLab, create Matrix rooms, send Matrix messages, -or acknowledge a downstream consumer. +These pure projectors turn bounded, normalized GitLab OTLP log fields into +factual CloudEvents. They do not call GitLab, choose local routing policy, +create Matrix rooms, send Matrix messages, or acknowledge a downstream +consumer. Numeric `gitlab.project.id` is the durable project identity; mutable +project paths are display and subject metadata only. -## Input compatibility +## Identity and compatibility -The production receiver exports the normalized event name as the OTLP log -attribute `event.name`. The original issue-created vertical also accepts the -OTLP LogRecord `eventName` field through the existing `signal:event.name` -fixture. New projectors prefer `attribute:event.name` and accept the signal -field as a compatibility fallback. +The source receiver supplies a stable GitLab delivery UUID in `event.id` and +`gitlab.event.id`. When one delivery becomes multiple facts, their occurrence +IDs are `:`. The OTLP adapter preserves that +occurrence ID and Maple derives the CloudEvent ID from tenant, source kind, +source, occurrence ID, projection ID, and projection revision. Payload hashes +are audit data, not identity. -All project and object IDs are retained as decimal strings because OTLP int64 -values are represented that way in the normalized signal model. Project IDs -and object IDs must be positive. Optional numeric durations and stage counts -must be non-negative. Strings are trimmed, rejected when blank, and limited to -4 KiB by the projector boundary. Raw webhook bodies, variables, URLs, secrets, -and confidential text are not projected. +New envelopes expose that preserved input as optional, backward-compatible +CloudEvents extensions `sourceoccurrenceid` and `sourceidentityquality`. +Historical envelopes lacking both fields remain schema-valid. A downstream +consumer can therefore persist the source occurrence ID, immutable Maple event +ID, and its own deterministic delivery transaction ID without parsing event +data. -## Version-1 contracts +The original `gitlab.issue.created@1` projector and +`dev.maple.gitlab.issue.created.v1` output remain unchanged. It accepts the +OTLP LogRecord `eventName` field used by its existing fixture. New projectors +prefer the `event.name` attribute and retain the signal field as a compatibility +fallback. -### `gitlab.project.lifecycle@1` +Complete output fixtures live in +`apps/cli/test/fixtures/gitlab-projectors.v1.json`. Their occurrence and +projection identity inputs are paired by index in +`apps/cli/test/fixtures/gitlab-projector-identities.v1.json`. -Input `event.name` values and normalized actions are: +## Common input and safety contract -| Input | `action` | +Every new projector requires: + +- `event.name`: one explicit event name documented below; +- `gitlab.project.id`: positive OTLP int64; +- `gitlab.project.path`: non-blank bounded text. + +Common optional fields are `gitlab.project.old_path`, `gitlab.actor.id`, +`gitlab.actor.name`, `gitlab.actor.username`, `gitlab.event.result`, and +resource `service.name`. Every output payload contains +`project: { id, path, oldPath? }`, `sourceEvent`, the relevant factual object, +and any present actor, result, and service name. + +The projector boundary enforces: + +- scalar text at most 4 KiB after trimming; +- comment excerpts at most 1,024 UTF-8 bytes after removing control + characters and normalizing whitespace; +- canonical HTTP(S) URLs without credentials or query strings, at most 2,048 + UTF-8 bytes; comment URLs may retain a note-anchor fragment; +- positive object IDs and non-negative counts/durations; +- hexadecimal revisions, including all-zero before/after revisions; +- at most 20 failed-job summaries and no unknown summary fields; +- no raw payloads, variables, logs, full comments, arbitrary descriptions, or + secrets. + +Projector config objects are closed and currently have no fields, except the +unchanged legacy issue-created `includeBody` option. + +## Version-1 event vocabulary + +### Project lifecycle + +Projector: `gitlab.project.lifecycle@1` + +Type: `dev.maple.gitlab.project.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-project-lifecycle:v1` + +| `event.name` | Output `action` | | -------------------------- | -------------------- | | `project_create` | `created` | -| `project_destroy` | `destroyed` | +| `project_update` | `updated` | | `project_rename` | `renamed` | | `project_transfer` | `transferred` | -| `project_update` | `updated` | | `project_archive` | `archived` | | `project_unarchive` | `unarchived` | | `project_deletion_request` | `deletion_requested` | +| `project_destroy` | `destroyed` | -Required fields are `gitlab.project.id`, `gitlab.project.path`, and -`event.name`. `gitlab.project.old_path`, `gitlab.actor.id`, -`gitlab.actor.name`, `gitlab.event.result`, and resource `service.name` are -optional. +`gitlab.project.old_path` is optional and normally present for rename or +transfer facts. Transfer direction is deliberately not inferred here. -The output type is `dev.maple.gitlab.project.lifecycle.v1`, with data shaped as: +### Issue lifecycle -```json -{ - "project": { "id": "42", "path": "rdev/maple", "oldPath": "rdev/old-maple" }, - "action": "renamed", - "sourceEvent": "project_rename", - "actor": { "id": "9", "name": "rdev" }, - "result": "success", - "serviceName": "gitlab-repository-events" -} -``` +Projector: `gitlab.issue.lifecycle@1` -### `gitlab.merge-request.lifecycle@1` +Type: `dev.maple.gitlab.issue.lifecycle.v1` -The source event must match `merge_request_`, where `` is a -bounded lowercase GitLab action token. The required fields are -`gitlab.project.id`, `gitlab.project.path`, `gitlab.merge_request.iid`, and -`event.name`. Source branch, target branch, merge commit, actor, result, and -resource service name are optional. +Schema: `urn:maple:event-schema:gitlab-issue-lifecycle:v1` -The output type is `dev.maple.gitlab.merge-request.lifecycle.v1`. Its subject -is `/-/merge_requests/` and its data has this shape: +| `event.name` | Output `issue.action` | +| -------------- | --------------------- | +| `issue_open` | `open` | +| `issue_update` | `update` | +| `issue_close` | `close` | +| `issue_reopen` | `reopen` | -```json -{ - "project": { "id": "42", "path": "rdev/maple" }, - "mergeRequest": { - "iid": "7", - "action": "open", - "sourceBranch": "feature/maple", - "targetBranch": "main", - "commit": "abc123" - }, - "sourceEvent": "merge_request_open", - "actor": { "id": "9", "name": "rdev" }, - "result": "success", - "serviceName": "gitlab-repository-events" -} -``` +Required: `gitlab.issue.iid`. Optional: positive `gitlab.issue.id`, +`gitlab.issue.title`, canonical `gitlab.issue.url`, and bounded lowercase +`gitlab.issue.state`. Structured `gitlab.issue.labels` is an optional array of +at most 50 non-blank labels, each at most 256 UTF-8 bytes. The same issue object, +including labels, is used by issue-comment events. + +### Issue comments + +Projector: `gitlab.issue.comment@1` + +Type: `dev.maple.gitlab.issue.comment.v1` + +Schema: `urn:maple:event-schema:gitlab-issue-comment:v1` + +The only accepted event is `issue_comment`. Required fields are +`gitlab.issue.iid` and positive `gitlab.comment.id`. Optional fields are +`gitlab.comment.excerpt`, canonical `gitlab.comment.url`, and boolean +`gitlab.comment.system`. The full comment body is never projected. + +### Merge-request lifecycle + +Projector: `gitlab.merge-request.lifecycle@1` + +Type: `dev.maple.gitlab.merge-request.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-merge-request-lifecycle:v1` + +| `event.name` | Output `mergeRequest.action` | +| ---------------------- | ---------------------------- | +| `merge_request_open` | `open` | +| `merge_request_update` | `update` | +| `merge_request_close` | `close` | +| `merge_request_reopen` | `reopen` | +| `merge_request_merge` | `merge` | +| `merge_request_review` | `review` | + +Required: positive `gitlab.merge_request.iid`. Optional: +`gitlab.merge_request.title`, canonical `gitlab.merge_request.url`, source and +target branches, hexadecimal `gitlab.merge_request.commit`, and bounded +`gitlab.merge_request.review_state`. Review events require `review_state`. +Unlike the initial implementation, arbitrary `merge_request_` events +fail closed. + +### Merge-request comments -### `gitlab.pipeline.completed@1` +Projector: `gitlab.merge-request.comment@1` -The source event must be `ci_pipeline_completed`. The required fields are -`gitlab.project.id`, `gitlab.project.path`, `gitlab.ci.pipeline.id`, -`gitlab.ci.pipeline.status`, and `event.name`. Status must be one of -`success`, `failed`, `canceled`, or `skipped`. Pipeline IID, name, source, -detailed status, ref, revision, durations, stage count, actor, result, and -resource service name are optional. +Type: `dev.maple.gitlab.merge-request.comment.v1` -The output type is `dev.maple.gitlab.pipeline.completed.v1`. Its subject is -`/-/pipelines/`: +Schema: `urn:maple:event-schema:gitlab-merge-request-comment:v1` + +`merge_request_comment` emits `comment.kind: "comment"` and +`merge_request_review_comment` emits `comment.kind: "review"`. Required fields +are `gitlab.merge_request.iid` and `gitlab.comment.id`; the same bounded comment +fields and sanitization rules as issue comments apply. + +### Completed pipelines + +Projector: `gitlab.pipeline.completed@1` + +Type: `dev.maple.gitlab.pipeline.completed.v1` + +Schema: `urn:maple:event-schema:gitlab-pipeline-completed:v1` + +The only accepted event is `ci_pipeline_completed`. Required fields are +positive `gitlab.ci.pipeline.id` and terminal +`gitlab.ci.pipeline.status: success|failed|canceled|skipped`. Optional scalar +fields are: + +- `gitlab.ci.pipeline.iid`, `name`, `source`, and `detailed_status`; +- positive `gitlab.ci.pipeline.merge_request_iid`, emitted only when the + pipeline-to-MR association is unambiguous; +- canonical `gitlab.ci.pipeline.url`; +- `vcs.ref` and hexadecimal `vcs.ref.head.revision`; +- non-negative `duration_ms`, `queued_duration_ms`, and `stage_count`; +- paired `failed_job_count` and boolean `failed_jobs_truncated`. + +The structured OTLP attribute `gitlab.ci.pipeline.failed_jobs` is an array of +at most 20 objects. Each object permits only positive string-encoded int64 +`id`, bounded `name`, optional `stage`, literal status `failed`, and optional +canonical `url`. When summaries are present, count and truncation metadata are +required and must agree with the array length. ```json { - "project": { "id": "42", "path": "rdev/maple" }, "pipeline": { "id": "900", - "iid": "12", "status": "failed", - "name": "Maple CI", - "source": "push", - "detailedStatus": "failed", - "ref": "main", - "sha": "deadbeef", - "durationMs": "63000", - "queuedDurationMs": "10000", - "stageCount": "3" - }, - "sourceEvent": "ci_pipeline_completed", - "actor": { "id": "9", "name": "rdev" }, - "result": "failure", - "serviceName": "gitlab-repository-events" + "url": "https://gitlab.internal/rdev/maple/-/pipelines/900", + "failedJobCount": "3", + "failedJobsTruncated": true, + "failedJobs": [ + { + "id": "901", + "name": "unit", + "stage": "test", + "status": "failed", + "url": "https://gitlab.internal/rdev/maple/-/jobs/901" + } + ] + } } ``` -## Complete CloudEvent fixtures +### Deployment lifecycle + +Projector: `gitlab.deployment.lifecycle@1` + +Type: `dev.maple.gitlab.deployment.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-deployment-lifecycle:v1` + +| `event.name` | Required `gitlab.deployment.status` | +| --------------------- | ----------------------------------- | +| `deployment_running` | `running` | +| `deployment_success` | `success` | +| `deployment_failed` | `failed` | +| `deployment_canceled` | `canceled` | +| `deployment_blocked` | `blocked` | +| `deployment_manual` | `manual` | + +Positive `gitlab.deployment.id`, bounded +`gitlab.deployment.environment`, and matching status are required. Optional +fields are hexadecimal `gitlab.deployment.revision` and canonical +`gitlab.deployment.url`. + +### Job lifecycle + +Projector: `gitlab.job.lifecycle@1` + +Type: `dev.maple.gitlab.job.lifecycle.v1` -These examples use the same source, occurrence IDs, projection IDs, and -revision as the compatibility tests. The IDs are the canonical Maple v1 -identity, so retrying the same source occurrence produces the same event ID. -The complete machine-readable set is checked in at -`apps/cli/test/fixtures/gitlab-projectors.v1.json`. +Schema: `urn:maple:event-schema:gitlab-job-lifecycle:v1` + +The event is `ci_job_` and `gitlab.ci.job.status` must match. Allowed +statuses are `created`, `pending`, `preparing`, `waiting_for_resource`, +`running`, `success`, `failed`, `canceled`, `skipped`, `manual`, and +`scheduled`. Positive `gitlab.ci.job.id`, bounded `gitlab.ci.job.name`, and +status are required. Optional fields are stage, canonical job URL, positive +pipeline ID, ref/revision, non-negative duration, and boolean `allow_failure`. + +### Ref lifecycle + +Projector: `gitlab.ref.lifecycle@1` + +Type: `dev.maple.gitlab.ref.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-ref-lifecycle:v1` + +The production receiver normalizes raw `push`, `tag_push`, and +`repository_update` deliveries into exactly six deduplicated semantic facts: + +| `event.name` | `ref.type` | `ref.action` | +| --------------- | ---------- | ------------ | +| `branch_create` | `branch` | `create` | +| `branch_update` | `branch` | `update` | +| `branch_delete` | `branch` | `delete` | +| `tag_create` | `tag` | `create` | +| `tag_update` | `tag` | `update` | +| `tag_delete` | `tag` | `delete` | + +Required fields are bounded `vcs.ref` and hexadecimal +`vcs.ref.base.revision`/`vcs.ref.head.revision`. All-zero revisions are valid. +Canonical `gitlab.ref.url` and non-negative `gitlab.push.commit_count` are +optional. Raw delivery names are not accepted by this projector. + +### Release lifecycle + +Projector: `gitlab.release.lifecycle@1` + +Type: `dev.maple.gitlab.release.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-release-lifecycle:v1` + +`release_create`, `release_update`, and `release_delete` map to matching +`release.action` values. Bounded `gitlab.release.tag` is required. Positive +`gitlab.release.id`, bounded name, and canonical URL are optional. Release +descriptions are intentionally excluded. + +## Complete sample CloudEvent + +The checked-in fixture set contains one complete envelope for each output +family and enriched variants where useful. For example: ```json { "specversion": "1.0", - "id": "sha256:0fac0a375c6f8a05fb5ec1a751cf9e3b55282f56c0d135bc8a8c50c6a2a7559f", + "id": "sha256:b7b29d2a12f061dc3e8b6c3bcb550302783b834541b6a8db7b104b4d14f41464", "source": "https://gitlab.internal", - "type": "dev.maple.gitlab.project.lifecycle.v1", - "subject": "rdev/maple", + "type": "dev.maple.gitlab.deployment.lifecycle.v1", + "subject": "rdev/maple/-/deployments/501", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", - "dataschema": "urn:maple:event-schema:gitlab-project-lifecycle:v1", + "dataschema": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", "tenantid": "local", - "projectionid": "gitlab-project-renamed", + "projectionid": "gitlab-deployment_failed", "projectionrevision": 1, - "projectorid": "gitlab.project.lifecycle", + "projectorid": "gitlab.deployment.lifecycle", "projectorversion": 1, + "sourceoccurrenceid": "delivery-deployment:deployment_failed", + "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple", "oldPath": "rdev/old-maple" }, - "action": "renamed", - "sourceEvent": "project_rename", - "actor": { "id": "9", "name": "rdev" }, - "result": "success", + "project": { "id": "42", "path": "rdev/maple" }, + "deployment": { + "id": "501", + "environment": "production", + "status": "failed", + "revision": "a8123fa", + "url": "https://gitlab.internal/rdev/maple/-/deployments/501" + }, + "sourceEvent": "deployment_failed", + "result": "failure", "serviceName": "gitlab-repository-events" } } ``` -The merge-request fixture uses -`sha256:09992cc6804e056fb2a037b6c9db05c432c4733e229c963675a15a47445f745b` -with occurrence ID `mr-event-7` and projection ID `gitlab-mr-opened`. The -pipeline fixture uses -`sha256:f2bab7e420aa6b97d65f0478d16384b45d3bf61fb9ad9c111c614279cb42e24e` -with occurrence ID `pipeline-event-900` and projection ID -`gitlab-pipeline-completed`; its status is `failed` and its normalized result -is `failure`. - -Downstream Matrix delivery may use these stable event IDs for idempotent -transaction IDs, but that consumer protocol is deliberately outside these -projectors. +Downstream delivery may derive an idempotent Matrix transaction ID from the +immutable Maple CloudEvent ID. That transaction policy and Matrix's structured +`info.selfenrichment.maple` content are downstream consumer contracts and do +not belong in these projectors. + +## Producer handoff + +The receiver must emit the explicit event vocabulary and fields above before a +corresponding projection is activated. In particular, a producer upgrade is +needed wherever the current receiver does not yet emit issue/comment facts, +MR title/URL/review/comment facts, pipeline canonical URL and bounded failed +jobs, deployment identity/environment/status/revision/URL, job lifecycle +fields, semantic ref transitions, or release facts. The receiver owns webhook +normalization, duplicate semantic-transition suppression, source UUID indexing, +excerpt pre-sanitization, and failed-job lookup/truncation. Maple validates and +projects those facts but does not reconstruct missing webhook data or call +GitLab. From f200809b5fb921a7169b61b3a139c24beda103f7 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 14 Aug 2026 12:32:39 -0400 Subject: [PATCH 12/12] feat(eventing): complete GitLab factual contracts --- apps/cli/src/server/eventing/control-store.ts | 399 ++++++++------ .../src/server/eventing/gitlab-projectors.ts | 61 ++- apps/cli/src/server/eventing/runtime.ts | 44 +- apps/cli/src/server/eventing/telemetry.ts | 80 +++ apps/cli/src/server/serve.ts | 22 +- .../test/fixtures/gitlab-projectors.v1.json | 144 ++--- apps/cli/test/gitlab-projectors.test.ts | 346 +++++++++++- .../test/local-eventing-control-store.test.ts | 70 +++ apps/cli/test/local-eventing-runtime.test.ts | 52 +- apps/cli/test/server-args.test.ts | 9 +- apps/cli/test/server-network.test.ts | 13 +- apps/local-ui/src/lib/constants.test.ts | 4 +- docs/gitlab-event-projectors.md | 124 +++-- docs/signal-to-event-projection.md | 10 + ...gitlab-deployment-lifecycle.v1.schema.json | 190 +++++++ .../gitlab-issue-comment.v1.schema.json | 280 ++++++++++ .../gitlab-issue-created.v1.schema.json | 62 +++ .../gitlab-issue-lifecycle.v1.schema.json | 234 ++++++++ .../gitlab-job-lifecycle.v1.schema.json | 249 +++++++++ ...itlab-merge-request-comment.v1.schema.json | 272 +++++++++ ...lab-merge-request-lifecycle.v1.schema.json | 226 ++++++++ .../gitlab-pipeline-completed.v1.schema.json | 518 ++++++++++++++++++ .../gitlab-project-lifecycle.v1.schema.json | 142 +++++ .../gitlab-ref-lifecycle.v1.schema.json | 208 +++++++ .../gitlab-release-lifecycle.v1.schema.json | 187 +++++++ .../eventing-core/scripts/generate-schemas.ts | 6 + packages/eventing-core/src/gitlab.ts | 297 ++++++++++ packages/eventing-core/src/index.ts | 1 + 28 files changed, 3906 insertions(+), 344 deletions(-) create mode 100644 apps/cli/src/server/eventing/telemetry.ts create mode 100644 packages/eventing-core/schemas/gitlab-deployment-lifecycle.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-issue-comment.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-issue-created.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-issue-lifecycle.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-job-lifecycle.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-merge-request-comment.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-merge-request-lifecycle.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-pipeline-completed.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-project-lifecycle.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-ref-lifecycle.v1.schema.json create mode 100644 packages/eventing-core/schemas/gitlab-release-lifecycle.v1.schema.json create mode 100644 packages/eventing-core/src/gitlab.ts diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts index 883163787..9f4aa5491 100644 --- a/apps/cli/src/server/eventing/control-store.ts +++ b/apps/cli/src/server/eventing/control-store.ts @@ -15,6 +15,7 @@ import { } from "@maple/eventing-core" import { Schema } from "effect" import { durableWrite, ensurePrivateDirectory } from "../durable-files" +import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" const CONTROL_SCHEMA_VERSION = 2 const CONTROL_DIRECTORY = "control" @@ -434,12 +435,19 @@ const decodeConsumer = (row: ConsumerRow): EventConsumer => ({ export class LocalEventingControlStore { readonly #db: Database readonly #limits: ResolvedLocalEventingControlLimits + readonly #telemetry: EventingTelemetry readonly path: string - private constructor(path: string, db: Database, limits: ResolvedLocalEventingControlLimits) { + private constructor( + path: string, + db: Database, + limits: ResolvedLocalEventingControlLimits, + telemetry: EventingTelemetry, + ) { this.path = path this.#db = db this.#limits = limits + this.#telemetry = telemetry } static async open( @@ -449,6 +457,7 @@ export class LocalEventingControlStore { maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, }, + telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, ): Promise { const validatedLimits = validateLimits(limits) const directory = eventingControlDirectory(dataDir) @@ -471,7 +480,7 @@ export class LocalEventingControlStore { ) chmodSync(path, 0o600) validateOpenDatabase(db) - return new LocalEventingControlStore(path, db, validatedLimits) + return new LocalEventingControlStore(path, db, validatedLimits, telemetry) } catch (error) { db.close() throw error @@ -557,80 +566,95 @@ export class LocalEventingControlStore { let inserted = 0 let deduplicated = 0 const eventIds: string[] = [] - this.#db - .transaction(() => { - const usage = this.#db - .query( - "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", - ) - .get() - if (!usage) throw new Error("event outbox usage query returned no row") - let outboxEvents = asNumber(usage.count) - let outboxBytes = asNumber(usage.bytes) - for (const candidate of events) { - const validated = validateMapleCloudEvent(candidate) - const { event, canonicalJson: eventJson, byteLength: eventBytes } = validated - const existing = this.#db - .query( - "SELECT event_id, event_json, state FROM outbox_events WHERE event_id = ?", - ) - .get(event.id) - if (existing) { - if (existing.event_json !== eventJson) - throw new Error(`event ID collision with different payload: ${event.id}`) - deduplicated += 1 - } else { - if ( - outboxEvents + 1 > this.#limits.maxOutboxEvents || - outboxBytes + eventBytes > this.#limits.maxOutboxBytes + try { + this.#db + .transaction(() => { + const usage = this.#db + .query( + "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", ) - throw new Error( - `event outbox capacity exceeded (${outboxEvents}/${this.#limits.maxOutboxEvents} events, ${outboxBytes}/${this.#limits.maxOutboxBytes} bytes)`, + .get() + if (!usage) throw new Error("event outbox usage query returned no row") + let outboxEvents = asNumber(usage.count) + let outboxBytes = asNumber(usage.bytes) + for (const candidate of events) { + const validated = validateMapleCloudEvent(candidate) + const { event, canonicalJson: eventJson, byteLength: eventBytes } = validated + const existing = this.#db + .query( + "SELECT event_id, event_json, state FROM outbox_events WHERE event_id = ?", ) - this.#db.run( - "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, state, event_json, staged_at) VALUES (?, ?, ?, ?, 'staged', ?, ?)", - [ - event.id, - event.tenantid, - event.projectionid, - event.projectionrevision, - eventJson, - stagedAt, - ], - ) - inserted += 1 - outboxEvents += 1 - outboxBytes += eventBytes + .get(event.id) + if (existing) { + if (existing.event_json !== eventJson) + throw new Error(`event ID collision with different payload: ${event.id}`) + deduplicated += 1 + } else { + if ( + outboxEvents + 1 > this.#limits.maxOutboxEvents || + outboxBytes + eventBytes > this.#limits.maxOutboxBytes + ) + throw new Error( + `event outbox capacity exceeded (${outboxEvents}/${this.#limits.maxOutboxEvents} events, ${outboxBytes}/${this.#limits.maxOutboxBytes} bytes)`, + ) + this.#db.run( + "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, state, event_json, staged_at) VALUES (?, ?, ?, ?, 'staged', ?, ?)", + [ + event.id, + event.tenantid, + event.projectionid, + event.projectionrevision, + eventJson, + stagedAt, + ], + ) + inserted += 1 + outboxEvents += 1 + outboxBytes += eventBytes + } + eventIds.push(event.id) } - eventIds.push(event.id) - } - }) - .immediate() + }) + .immediate() + } catch (error) { + this.#telemetry.record({ operation: "outbox_stage", outcome: "failure" }) + throw error + } + this.#telemetry.record({ operation: "outbox_stage", outcome: "success", count: inserted }) + this.#telemetry.record({ operation: "outbox_dedup", outcome: "success", count: deduplicated }) return { inserted, deduplicated, eventIds } } markReady(eventIds: readonly string[], readyAt = new Date().toISOString()): void { - this.#db - .transaction(() => { - for (const eventId of eventIds) { - const row = this.#db - .query, [string]>( - "SELECT state FROM outbox_events WHERE event_id = ?", + let markedReady = 0 + try { + this.#db + .transaction(() => { + for (const eventId of eventIds) { + const row = this.#db + .query, [string]>( + "SELECT state FROM outbox_events WHERE event_id = ?", + ) + .get(eventId) + if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) + if (row.state === "ready") continue + this.#db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ + eventId, + readyAt, + ]) + this.#db.run( + "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", + [readyAt, eventId], ) - .get(eventId) - if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) - if (row.state === "ready") continue - this.#db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ - eventId, - readyAt, - ]) - this.#db.run( - "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", - [readyAt, eventId], - ) - } - }) - .immediate() + markedReady += 1 + } + }) + .immediate() + } catch (error) { + this.#telemetry.record({ operation: "outbox_ready", outcome: "failure" }) + throw error + } + this.#telemetry.record({ operation: "outbox_ready", outcome: "success", count: markedReady }) } #listOutbox(state: "ready" | "staged", limit = 100, after = 0): EventingOutboxPage { @@ -765,74 +789,95 @@ export class LocalEventingControlStore { leaseSeconds: number, now = new Date().toISOString(), ): EventConsumerClaim { - validateConsumerId(consumerId) - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) - throw new EventConsumerInputError("claim limit must be between 1 and 1000") - if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) - throw new EventConsumerInputError("leaseSeconds must be between 5 and 300") - const nowMilliseconds = canonicalInstant(now, "claim time") - return this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (!consumer) throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) - if (asNumber(consumer.active) === 0) - throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) - if ( - consumer.lease_expires_at !== null && - canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > - nowMilliseconds - ) - throw new EventConsumerConflictError( - `event consumer already has an active lease: ${consumerId}`, + let reclaimedExpiredLease = false + let lag = 0 + try { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new EventConsumerInputError("claim limit must be between 1 and 1000") + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) + throw new EventConsumerInputError("leaseSeconds must be between 5 and 300") + const nowMilliseconds = canonicalInstant(now, "claim time") + const claim = this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) + throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_expires_at !== null && + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > + nowMilliseconds ) + throw new EventConsumerConflictError( + `event consumer already has an active lease: ${consumerId}`, + ) + if (consumer.lease_expires_at !== null) reclaimedExpiredLease = true + lag = this.#consumerLag(tenantId, asNumber(consumer.last_acked_sequence)) - const rows = this.#db - .query( - `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + const rows = this.#db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at FROM outbox_ready_events AS readiness INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id WHERE event.tenant_id = ? AND event.state = 'ready' AND readiness.sequence > ? ORDER BY readiness.sequence LIMIT ?`, - ) - .all(tenantId, asNumber(consumer.last_acked_sequence), limit) - if (rows.length === 0) { + ) + .all(tenantId, asNumber(consumer.last_acked_sequence), limit) + if (rows.length === 0) { + this.#db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", + [tenantId, consumerId], + ) + return { + consumerId, + leaseToken: null, + leaseExpiresAt: null, + throughSequence: null, + events: [], + } + } + + const leaseToken = randomBytes(32).toString("hex") + const leaseExpiresAt = new Date(nowMilliseconds + leaseSeconds * 1_000).toISOString() + const throughSequence = asNumber(rows.at(-1)!.sequence) this.#db.run( - "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", - [tenantId, consumerId], + `UPDATE event_consumers + SET lease_token_hash = ?, lease_expires_at = ?, claimed_through_sequence = ? + WHERE tenant_id = ? AND consumer_id = ?`, + [tokenHash(leaseToken), leaseExpiresAt, throughSequence, tenantId, consumerId], ) return { consumerId, - leaseToken: null, - leaseExpiresAt: null, - throughSequence: null, - events: [], + leaseToken, + leaseExpiresAt, + throughSequence, + events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })), } - } - - const leaseToken = randomBytes(32).toString("hex") - const leaseExpiresAt = new Date(nowMilliseconds + leaseSeconds * 1_000).toISOString() - const throughSequence = asNumber(rows.at(-1)!.sequence) - this.#db.run( - `UPDATE event_consumers - SET lease_token_hash = ?, lease_expires_at = ?, claimed_through_sequence = ? - WHERE tenant_id = ? AND consumer_id = ?`, - [tokenHash(leaseToken), leaseExpiresAt, throughSequence, tenantId, consumerId], - ) - return { - consumerId, - leaseToken, - leaseExpiresAt, - throughSequence, - events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ - sequence: asNumber(sequence), - event: decodeEvent(event_json), - stagedAt: staged_at, - readyAt: ready_at, - })), - } + }) + .immediate() + this.#telemetry.record({ + operation: "consumer_claim", + outcome: claim.events.length === 0 ? "empty" : "success", + count: Math.max(1, claim.events.length), }) - .immediate() + this.#telemetry.record({ operation: "consumer_lag", outcome: "observed", lag }) + if (reclaimedExpiredLease) + this.#telemetry.record({ operation: "consumer_lease", outcome: "reclaimed" }) + return claim + } catch (error) { + this.#telemetry.record({ operation: "consumer_claim", outcome: "failure" }) + if (error instanceof EventConsumerConflictError && /lease/.test(error.message)) + this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) + throw error + } } acknowledgeClaim( @@ -842,48 +887,82 @@ export class LocalEventingControlStore { throughSequence: number, now = new Date().toISOString(), ): EventConsumerAcknowledgement { - validateConsumerId(consumerId) - if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) - throw new EventConsumerInputError("throughSequence must be a positive safe integer") - const nowMilliseconds = canonicalInstant(now, "acknowledgement time") - return this.#db - .transaction(() => { - const consumer = this.#consumer(tenantId, consumerId) - if (!consumer) throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) - if (asNumber(consumer.active) === 0) - throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) - if ( - consumer.lease_token_hash === null || - consumer.lease_expires_at === null || - consumer.claimed_through_sequence === null - ) - throw new EventConsumerConflictError(`event consumer has no active lease: ${consumerId}`) - if ( - canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") <= - nowMilliseconds - ) - throw new EventConsumerConflictError(`event consumer lease has expired: ${consumerId}`) - if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) - throw new EventConsumerConflictError("event consumer lease token does not match") - const claimedThrough = asNumber(consumer.claimed_through_sequence) - if (throughSequence !== claimedThrough) - throw new EventConsumerConflictError( - `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, + try { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) + throw new EventConsumerInputError("throughSequence must be a positive safe integer") + const nowMilliseconds = canonicalInstant(now, "acknowledgement time") + const acknowledgement = this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) + throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_token_hash === null || + consumer.lease_expires_at === null || + consumer.claimed_through_sequence === null ) - this.#db.run( - `UPDATE event_consumers + throw new EventConsumerConflictError( + `event consumer has no active lease: ${consumerId}`, + ) + if ( + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") <= + nowMilliseconds + ) + throw new EventConsumerConflictError( + `event consumer lease has expired: ${consumerId}`, + ) + if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) + throw new EventConsumerConflictError("event consumer lease token does not match") + const claimedThrough = asNumber(consumer.claimed_through_sequence) + if (throughSequence !== claimedThrough) + throw new EventConsumerConflictError( + `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, + ) + this.#db.run( + `UPDATE event_consumers SET last_acked_sequence = ?, lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?`, - [throughSequence, tenantId, consumerId], - ) - return { - consumerId, - acknowledgedThrough: throughSequence, - prunedEvents: this.#pruneAcknowledgedReady(tenantId), - } + [throughSequence, tenantId, consumerId], + ) + return { + consumerId, + acknowledgedThrough: throughSequence, + prunedEvents: this.#pruneAcknowledgedReady(tenantId), + } + }) + .immediate() + this.#telemetry.record({ operation: "consumer_ack", outcome: "success" }) + this.#telemetry.record({ + operation: "consumer_lag", + outcome: "observed", + lag: this.#consumerLag(tenantId, acknowledgement.acknowledgedThrough), }) - .immediate() + return acknowledgement + } catch (error) { + this.#telemetry.record({ operation: "consumer_ack", outcome: "failure" }) + if (error instanceof EventConsumerConflictError && /lease/.test(error.message)) + this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) + throw error + } + } + + #consumerLag(tenantId: string, lastAcknowledgedSequence: number): number { + const latest = this.#db + .query( + `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready'`, + ) + .get(tenantId) + return Math.max( + 0, + (latest?.sequence == null ? 0 : asNumber(latest.sequence)) - lastAcknowledgedSequence, + ) } #consumer(tenantId: string, consumerId: string): ConsumerRow | null { diff --git a/apps/cli/src/server/eventing/gitlab-projectors.ts b/apps/cli/src/server/eventing/gitlab-projectors.ts index 39b3796dc..c1ee2a9f8 100644 --- a/apps/cli/src/server/eventing/gitlab-projectors.ts +++ b/apps/cli/src/server/eventing/gitlab-projectors.ts @@ -292,6 +292,11 @@ const eventName = (signal: NormalizedSignal): string => true, )! +const requireSourceProvidedIdentity = (signal: NormalizedSignal): void => { + if (signal.identityQuality !== "source" || signal.occurrenceId === null) + throw new Error("GitLab projector requires source-provided occurrence identity") +} + const actor = (signal: NormalizedSignal) => { const id = optionalPositiveProjectorInt64( field(signal, "attribute", "gitlab.actor.id"), @@ -319,10 +324,12 @@ const issue = (signal: NormalizedSignal) => { const title = projectorString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") const url = projectorUrl(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") const state = projectorToken(field(signal, "attribute", "gitlab.issue.state"), "gitlab.issue.state") + const type = projectorToken(field(signal, "attribute", "gitlab.issue.type"), "gitlab.issue.type", true)! const labels = issueLabels(signal) return { ...(id === undefined ? {} : { id }), iid, + type, ...(title === undefined ? {} : { title }), ...(url === undefined ? {} : { url }), ...(state === undefined ? {} : { state }), @@ -338,11 +345,13 @@ const mergeRequest = (signal: NormalizedSignal) => { const title = projectorString( field(signal, "attribute", "gitlab.merge_request.title"), "gitlab.merge_request.title", - ) + true, + )! const url = projectorUrl( field(signal, "attribute", "gitlab.merge_request.url"), "gitlab.merge_request.url", - ) + true, + )! const sourceBranch = projectorString( field(signal, "attribute", "gitlab.merge_request.source_branch"), "gitlab.merge_request.source_branch", @@ -361,8 +370,8 @@ const mergeRequest = (signal: NormalizedSignal) => { ) return { iid, - ...(title === undefined ? {} : { title }), - ...(url === undefined ? {} : { url }), + title, + url, ...(sourceBranch === undefined ? {} : { sourceBranch }), ...(targetBranch === undefined ? {} : { targetBranch }), ...(commit === undefined ? {} : { commit }), @@ -376,12 +385,13 @@ const comment = (signal: NormalizedSignal, kind?: "comment" | "review") => { field(signal, "attribute", "gitlab.comment.excerpt"), "gitlab.comment.excerpt", ) + if (excerpt === undefined) throw new Error("GitLab projector is missing gitlab.comment.excerpt") const url = projectorUrl( field(signal, "attribute", "gitlab.comment.url"), "gitlab.comment.url", - false, true, - ) + true, + )! const system = projectorBoolean( field(signal, "attribute", "gitlab.comment.system"), "gitlab.comment.system", @@ -389,8 +399,8 @@ const comment = (signal: NormalizedSignal, kind?: "comment" | "review") => { return { id, ...(kind === undefined ? {} : { kind }), - ...(excerpt === undefined ? {} : { excerpt }), - ...(url === undefined ? {} : { url }), + excerpt, + url, ...(system === undefined ? {} : { system }), } } @@ -455,6 +465,7 @@ const failedJobs = (signal: NormalizedSignal) => { } const project = (signal: NormalizedSignal) => { + requireSourceProvidedIdentity(signal) const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") const path = projectorString( field(signal, "attribute", "gitlab.project.path"), @@ -698,9 +709,10 @@ export const gitlabPipelineCompletedProjector = { const url = projectorUrl( field(signal, "attribute", "gitlab.ci.pipeline.url"), "gitlab.ci.pipeline.url", - ) + true, + )! const ref = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref") - const sha = projectorString( + const sha = projectorRevision( field(signal, "attribute", "vcs.ref.head.revision"), "vcs.ref.head.revision", ) @@ -725,19 +737,23 @@ export const gitlabPipelineCompletedProjector = { "gitlab.ci.pipeline.failed_jobs_truncated", ) const pipelineFailedJobs = failedJobs(signal) - if ((failedJobCount === undefined) !== (failedJobsTruncated === undefined)) - throw new Error("GitLab projector failed job count and truncation flag must be supplied together") - if (pipelineFailedJobs !== undefined) { - if (status !== "failed") - throw new Error("GitLab projector failed jobs are only valid for a failed pipeline") + if (status === "failed") { if (failedJobCount === undefined) throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_job_count") if (failedJobsTruncated === undefined) throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_jobs_truncated") + if (pipelineFailedJobs === undefined) + throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_jobs") if (BigInt(failedJobCount) < BigInt(pipelineFailedJobs.length)) throw new Error("GitLab projector failed job count is smaller than the summary list") if (failedJobsTruncated !== BigInt(failedJobCount) > BigInt(pipelineFailedJobs.length)) throw new Error("GitLab projector failed job truncation flag is inconsistent") + } else if ( + failedJobCount !== undefined || + failedJobsTruncated !== undefined || + pipelineFailedJobs !== undefined + ) { + throw new Error("GitLab projector failed job summaries are only valid for a failed pipeline") } const eventActor = actor(signal) const eventServiceName = serviceName(signal) @@ -754,7 +770,7 @@ export const gitlabPipelineCompletedProjector = { ...(name === undefined ? {} : { name }), ...(pipelineSource === undefined ? {} : { source: pipelineSource }), ...(detailedStatus === undefined ? {} : { detailedStatus }), - ...(url === undefined ? {} : { url }), + url, ...(ref === undefined ? {} : { ref }), ...(sha === undefined ? {} : { sha }), ...(durationMs === undefined ? {} : { durationMs }), @@ -805,8 +821,13 @@ export const gitlabDeploymentLifecycleProjector = { const revision = projectorRevision( field(signal, "attribute", "gitlab.deployment.revision"), "gitlab.deployment.revision", - ) - const url = projectorUrl(field(signal, "attribute", "gitlab.deployment.url"), "gitlab.deployment.url") + true, + )! + const url = projectorUrl( + field(signal, "attribute", "gitlab.deployment.url"), + "gitlab.deployment.url", + true, + )! const eventActor = actor(signal) const eventResult = result(signal) const eventServiceName = serviceName(signal) @@ -818,8 +839,8 @@ export const gitlabDeploymentLifecycleProjector = { id, environment, status, - ...(revision === undefined ? {} : { revision }), - ...(url === undefined ? {} : { url }), + revision, + url, }, sourceEvent, ...(eventActor === undefined ? {} : { actor: eventActor }), diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts index b3542abe3..0da077b85 100644 --- a/apps/cli/src/server/eventing/runtime.ts +++ b/apps/cli/src/server/eventing/runtime.ts @@ -13,6 +13,7 @@ import { LocalEventingControlStore } from "./control-store" import type { EventConsumerStart } from "./control-store" import { registerGitLabProjectors } from "./gitlab-projectors" import { OTLP_LOG_ADAPTER } from "./otlp" +import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" const TENANT_ID = "local" @@ -39,12 +40,14 @@ export class LocalEventingRuntime { readonly #store: LocalEventingControlStore readonly #sources: SignalSourceRegistry readonly #projectors: ProjectorRegistry + readonly #telemetry: EventingTelemetry #compiled: CompiledProjectionRegistry #activeSourceKinds = new Set() #generation = 0 - constructor(store: LocalEventingControlStore) { + constructor(store: LocalEventingControlStore, telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY) { this.#store = store + this.#telemetry = telemetry this.#sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) this.#projectors = registerGitLabProjectors(new ProjectorRegistry()) const specs = store.loadEnabledProjections(TENANT_ID) @@ -93,16 +96,49 @@ export class LocalEventingRuntime { ): LocalProjectionEvaluation { const sourceKind = signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" if (!this.hasActiveSource(sourceKind)) return emptyEvaluation() + const startedAt = performance.now() const acceptedAt = new Date().toISOString() - const normalized = ( - signal === "logs" ? OTLP_LOG_ADAPTER.normalize(decoded, { acceptedAt, tenantId: TENANT_ID }) : [] - ).filter((occurrence) => !isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) + let normalized + try { + normalized = ( + signal === "logs" + ? OTLP_LOG_ADAPTER.normalize(decoded, { acceptedAt, tenantId: TENANT_ID }) + : [] + ).filter((occurrence) => !isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) + this.#telemetry.record({ + operation: "normalization", + outcome: "success", + count: normalized.length, + durationMs: performance.now() - startedAt, + sourceKind, + }) + } catch (error) { + this.#telemetry.record({ + operation: "normalization", + outcome: "failure", + durationMs: performance.now() - startedAt, + sourceKind, + }) + throw error + } const snapshot = this.#compiled const events: MapleCloudEvent[] = [] const failures: ProjectionFailure[] = [] const typeMismatchFields = new Set() for (const occurrence of normalized) { const result = snapshot.evaluate(occurrence) + this.#telemetry.record({ + operation: "projection", + outcome: "success", + count: result.events.length, + sourceKind, + }) + this.#telemetry.record({ + operation: "projection", + outcome: "failure", + count: result.failures.length, + sourceKind, + }) events.push(...result.events) failures.push(...result.failures) for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) diff --git a/apps/cli/src/server/eventing/telemetry.ts b/apps/cli/src/server/eventing/telemetry.ts new file mode 100644 index 000000000..98ba74cbf --- /dev/null +++ b/apps/cli/src/server/eventing/telemetry.ts @@ -0,0 +1,80 @@ +import { Effect, Metric } from "effect" + +export type EventingTelemetryOperation = + | "normalization" + | "projection" + | "outbox_stage" + | "outbox_ready" + | "outbox_dedup" + | "consumer_claim" + | "consumer_ack" + | "consumer_lease" + | "consumer_lag" + +export type EventingTelemetryOutcome = + | "success" + | "failure" + | "empty" + | "active" + | "expired" + | "reclaimed" + | "observed" + +export type EventingTelemetrySourceKind = "otel.log" | "otel.span" | "otel.metric" | "unknown" + +/** Deliberately excludes tenant, consumer, event, projection, payload, and credential values. */ +export interface EventingTelemetryObservation { + readonly operation: EventingTelemetryOperation + readonly outcome: EventingTelemetryOutcome + readonly count?: number + readonly durationMs?: number + readonly lag?: number + readonly sourceKind?: EventingTelemetrySourceKind +} + +export interface EventingTelemetry { + record(observation: EventingTelemetryObservation): void +} + +export const NOOP_EVENTING_TELEMETRY: EventingTelemetry = { record: () => {} } + +const operations = Metric.counter("maple.eventing.operations_total", { + description: "Eventing operations by bounded operation and outcome", + incremental: true, +}) +const durations = Metric.histogram("maple.eventing.operation_duration_ms", { + description: "Eventing operation duration in milliseconds", + boundaries: [0.1, 0.5, 1, 5, 10, 50, 100, 500, 1_000, 5_000], +}) +const consumerLag = Metric.histogram("maple.eventing.consumer_lag_events", { + description: "Ready-event sequence lag observed by event consumers", + boundaries: [0, 1, 5, 10, 50, 100, 500, 1_000, 10_000], +}) + +export const makeEffectEventingTelemetry = ( + run: (effect: Effect.Effect) => void, +): EventingTelemetry => ({ + record(observation) { + const attributes = { + operation: observation.operation, + outcome: observation.outcome, + source_kind: observation.sourceKind ?? "unknown", + } + const effects: Effect.Effect[] = [] + const count = observation.count ?? 1 + if (Number.isFinite(count) && count > 0) + effects.push(Metric.update(Metric.withAttributes(operations, attributes), count)) + if (observation.durationMs !== undefined && Number.isFinite(observation.durationMs)) + effects.push( + Metric.update( + Metric.withAttributes(durations, attributes), + Math.max(0, observation.durationMs), + ), + ) + if (observation.lag !== undefined && Number.isSafeInteger(observation.lag)) + effects.push( + Metric.update(Metric.withAttributes(consumerLag, attributes), Math.max(0, observation.lag)), + ) + if (effects.length > 0) run(Effect.all(effects, { discard: true })) + }, +}) diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index f903b0073..d2b8c17a2 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -26,6 +26,7 @@ import { } from "./eventing/control-store" import { ensureEventConsumerToken, eventConsumerTokenMatches } from "./eventing/consumer-auth" import { LocalEventingRuntime } from "./eventing/runtime" +import { makeEffectEventingTelemetry } from "./eventing/telemetry" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { decodeLogsRequest, @@ -993,9 +994,18 @@ export const startServer = ( configFile: options.configFile, rawTelemetryRetentionDays: retention.effective, }) + // The request handler and synchronous eventing store share one telemetry + // runtime; eventing observations contain only bounded operation labels. + const telemetry = yield* Effect.acquireRelease( + Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), + (rt) => Effect.promise(() => rt.dispose()), + ) + const eventingTelemetry = makeEffectEventingTelemetry((effect) => { + telemetry.runFork(effect) + }) const controlStore = yield* Effect.acquireRelease( Effect.tryPromise({ - try: () => LocalEventingControlStore.open(options.dataDir), + try: () => LocalEventingControlStore.open(options.dataDir, undefined, eventingTelemetry), catch: (error) => new ChdbError({ message: `failed to open local eventing control store: ${error instanceof Error ? error.message : String(error)}`, @@ -1004,7 +1014,7 @@ export const startServer = ( (store) => Effect.sync(() => store.close()), ) const eventing = yield* Effect.try({ - try: () => new LocalEventingRuntime(controlStore), + try: () => new LocalEventingRuntime(controlStore, eventingTelemetry), catch: (error) => new ChdbError({ message: `failed to compile local event projections: ${error instanceof Error ? error.message : String(error)}`, @@ -1076,14 +1086,6 @@ export const startServer = ( }), }) const gate = new RequestQuiescenceGate() - // A dedicated runtime carrying the OTel tracer for per-request spans: the - // Bun.serve handler runs outside Effect, so each request's span effect is - // run through this runtime. Disposed on scope close, which flushes any - // pending spans (bounded by the layer's shutdownTimeout). - const telemetry = yield* Effect.acquireRelease( - Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), - (rt) => Effect.promise(() => rt.dispose()), - ) const runSpan: SpanRunner = (effect) => telemetry.runPromise(effect) const server = yield* Effect.acquireRelease( Effect.try({ diff --git a/apps/cli/test/fixtures/gitlab-projectors.v1.json b/apps/cli/test/fixtures/gitlab-projectors.v1.json index 02c223964..d4bf87b6c 100644 --- a/apps/cli/test/fixtures/gitlab-projectors.v1.json +++ b/apps/cli/test/fixtures/gitlab-projectors.v1.json @@ -1,10 +1,10 @@ [ { "specversion": "1.0", - "id": "sha256:0fac0a375c6f8a05fb5ec1a751cf9e3b55282f56c0d135bc8a8c50c6a2a7559f", - "source": "https://gitlab.internal", + "id": "sha256:fc797f0e2390dbb69749b2718831dd31f18f0412112e0d90d19fc83bf7a092ba", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.project.lifecycle.v1", - "subject": "rdev/maple", + "subject": "example/widgets", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-project-lifecycle:v1", @@ -16,20 +16,20 @@ "sourceoccurrenceid": "project-event-42", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple", "oldPath": "rdev/old-maple" }, + "project": { "id": "42", "path": "example/widgets", "oldPath": "example/old-widgets" }, "action": "renamed", "sourceEvent": "project_rename", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "success", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:09992cc6804e056fb2a037b6c9db05c432c4733e229c963675a15a47445f745b", - "source": "https://gitlab.internal", + "id": "sha256:810de97620b3c54c274b9052ef209b2a28e15d8879b88e69125c49d10aec51fe", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.merge-request.lifecycle.v1", - "subject": "rdev/maple/-/merge_requests/7", + "subject": "example/widgets/-/merge_requests/7", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", @@ -41,26 +41,28 @@ "sourceoccurrenceid": "mr-event-7", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "mergeRequest": { "iid": "7", + "title": "Freeze contracts", + "url": "https://gitlab.example.test/example/widgets/-/merge_requests/7", "action": "open", "sourceBranch": "feature/maple", "targetBranch": "main", "commit": "abc123" }, "sourceEvent": "merge_request_open", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "success", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:f2bab7e420aa6b97d65f0478d16384b45d3bf61fb9ad9c111c614279cb42e24e", - "source": "https://gitlab.internal", + "id": "sha256:de495a092df476adba1f53296ec3f7b2966de886bf78aca7937109dc43f86ad9", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.pipeline.completed.v1", - "subject": "rdev/maple/-/pipelines/900", + "subject": "example/widgets/-/pipelines/900", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-pipeline-completed:v1", @@ -72,7 +74,7 @@ "sourceoccurrenceid": "pipeline-event-900", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "pipeline": { "id": "900", "iid": "12", @@ -80,24 +82,28 @@ "name": "Maple CI", "source": "push", "detailedStatus": "failed", + "url": "https://gitlab.example.test/example/widgets/-/pipelines/900", "ref": "main", "sha": "deadbeef", "durationMs": "63000", "queuedDurationMs": "10000", - "stageCount": "3" + "stageCount": "3", + "failedJobCount": "0", + "failedJobsTruncated": false, + "failedJobs": [] }, "sourceEvent": "ci_pipeline_completed", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "failure", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:880765fe8243154db7a7c1902ff328ca360da3acda28c9081539e35aea16798b", - "source": "https://gitlab.internal", + "id": "sha256:7311f08794a75eefba39bf98beb0533e8d30510525c20bf379fd1594618dad76", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.issue.lifecycle.v1", - "subject": "rdev/maple/-/issues/7", + "subject": "example/widgets/-/issues/7", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-issue-lifecycle:v1", @@ -109,27 +115,28 @@ "sourceoccurrenceid": "delivery-issue:issue_close", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "issue": { "id": "70", "iid": "7", + "type": "issue", "title": "Typed events", - "url": "https://gitlab.internal/rdev/maple/-/issues/7", + "url": "https://gitlab.example.test/example/widgets/-/issues/7", "labels": ["agent-ready", "backend"], "action": "close" }, "sourceEvent": "issue_close", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "success", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:b3e27095a6aa053a50b2842109bb0a48ec5140db48ae6af1f13352ae14dbe515", - "source": "https://gitlab.internal", + "id": "sha256:d4e689dbe8e01d81500acd6ba67eb9f9dea7a251c24b8d852b2fd44573feb2f9", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.issue.comment.v1", - "subject": "rdev/maple/-/issues/7#note_81", + "subject": "example/widgets/-/issues/7#note_81", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-issue-comment:v1", @@ -141,25 +148,25 @@ "sourceoccurrenceid": "delivery-comment:0", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, - "issue": { "iid": "7", "labels": ["agent-ready", "backend"] }, + "project": { "id": "42", "path": "example/widgets" }, + "issue": { "iid": "7", "type": "issue", "labels": ["agent-ready", "backend"] }, "comment": { "id": "81", "excerpt": "hello Maple", - "url": "https://gitlab.internal/rdev/maple/-/issues/7#note_81" + "url": "https://gitlab.example.test/example/widgets/-/issues/7#note_81" }, "sourceEvent": "issue_comment", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "success", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:6e2f7dd052deee38d40f3f0dc37a3df0f42e753c322d65be0eaed74ced497016", - "source": "https://gitlab.internal", + "id": "sha256:b2ea1f0f683074c6fb5ce451427c873f797b2425af0fd7924117b7956dd646c3", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.merge-request.comment.v1", - "subject": "rdev/maple/-/merge_requests/7#note_82", + "subject": "example/widgets/-/merge_requests/7#note_82", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-merge-request-comment:v1", @@ -171,21 +178,30 @@ "sourceoccurrenceid": "delivery-mr-comment:merge_request_review_comment", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, - "mergeRequest": { "iid": "7" }, - "comment": { "id": "82", "kind": "review", "excerpt": "Please add a fixture" }, + "project": { "id": "42", "path": "example/widgets" }, + "mergeRequest": { + "iid": "7", + "title": "Freeze contracts", + "url": "https://gitlab.example.test/example/widgets/-/merge_requests/7" + }, + "comment": { + "id": "82", + "kind": "review", + "excerpt": "Please add a fixture", + "url": "https://gitlab.example.test/example/widgets/-/merge_requests/7#note_82" + }, "sourceEvent": "merge_request_review_comment", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "success", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:7c9304b99e00f9d8be2e758bf8b98c78387f22be23f415810ec3d44fed3d4c89", - "source": "https://gitlab.internal", + "id": "sha256:9ccbec35baf0a0898aa80e8b7ef29dcb294dbdbb060b895224d09f00cda2c74c", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.pipeline.completed.v1", - "subject": "rdev/maple/-/pipelines/900", + "subject": "example/widgets/-/pipelines/900", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-pipeline-completed:v1", @@ -197,12 +213,12 @@ "sourceoccurrenceid": "delivery-pipeline:0", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "pipeline": { "id": "900", "mergeRequestIid": "7", "status": "failed", - "url": "https://gitlab.internal/rdev/maple/-/pipelines/900", + "url": "https://gitlab.example.test/example/widgets/-/pipelines/900", "failedJobCount": "3", "failedJobsTruncated": true, "failedJobs": [ @@ -211,23 +227,23 @@ "name": "unit", "stage": "test", "status": "failed", - "url": "https://gitlab.internal/rdev/maple/-/jobs/901" + "url": "https://gitlab.example.test/example/widgets/-/jobs/901" }, { "id": "902", "name": "integration", "stage": "test", "status": "failed" } ] }, "sourceEvent": "ci_pipeline_completed", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "failure", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:b7b29d2a12f061dc3e8b6c3bcb550302783b834541b6a8db7b104b4d14f41464", - "source": "https://gitlab.internal", + "id": "sha256:7f8aea4bc391d3b954fecd9ef5356b02aaeb4289ddfa25ef9c3c16217b0d77d1", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.deployment.lifecycle.v1", - "subject": "rdev/maple/-/deployments/501", + "subject": "example/widgets/-/deployments/501", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", @@ -239,26 +255,26 @@ "sourceoccurrenceid": "delivery-deployment:deployment_failed", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "deployment": { "id": "501", "environment": "production", "status": "failed", "revision": "a8123fa", - "url": "https://gitlab.internal/rdev/maple/-/deployments/501" + "url": "https://gitlab.example.test/example/widgets/-/deployments/501" }, "sourceEvent": "deployment_failed", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "failure", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:88af5c5523b40f64488a3fa8134139813974973ca826197006d414802fad03fe", - "source": "https://gitlab.internal", + "id": "sha256:42b44cd2784fb2e33dd5484b430cc11f429182db33907ca47e882631ddfdd18a", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.job.lifecycle.v1", - "subject": "rdev/maple/-/jobs/901", + "subject": "example/widgets/-/jobs/901", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-job-lifecycle:v1", @@ -270,20 +286,20 @@ "sourceoccurrenceid": "delivery-job:failed", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "job": { "id": "901", "name": "unit", "status": "failed" }, "sourceEvent": "ci_job_failed", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "failure", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:30adaaa10eb7d5dace8a2440e6b110f06d77948fe605d3b6cb274d8b2433b804", - "source": "https://gitlab.internal", + "id": "sha256:8c3d03bcb1a4cd856324477d95f9ffbf3702aa3dc9a103c3995071eb0ac811e0", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.ref.lifecycle.v1", - "subject": "rdev/maple/-/refs/refs/tags/v1.0.0", + "subject": "example/widgets/-/refs/refs/tags/v1.0.0", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-ref-lifecycle:v1", @@ -295,7 +311,7 @@ "sourceoccurrenceid": "delivery-ref:tag_create", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "ref": { "type": "tag", "action": "create", @@ -304,17 +320,17 @@ "after": "a8123faa8123faa8123faa8123faa8123faa8123" }, "sourceEvent": "tag_create", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "success", "serviceName": "gitlab-repository-events" } }, { "specversion": "1.0", - "id": "sha256:d784c81bef9413f2132218d086ab15f81ec49ef9ec833becb622f92fac58f617", - "source": "https://gitlab.internal", + "id": "sha256:9f141e93b5445145b4b7e2f88c3cfe8d53055a4d7ccf88df365415f96d43f0b5", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.release.lifecycle.v1", - "subject": "rdev/maple/-/releases/v1.0.0", + "subject": "example/widgets/-/releases/v1.0.0", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-release-lifecycle:v1", @@ -326,10 +342,10 @@ "sourceoccurrenceid": "delivery-release:create", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "release": { "tag": "v1.0.0", "action": "create", "name": "Maple 1.0" }, "sourceEvent": "release_create", - "actor": { "id": "9", "name": "rdev" }, + "actor": { "id": "9", "name": "maintainer" }, "result": "success", "serviceName": "gitlab-repository-events" } diff --git a/apps/cli/test/gitlab-projectors.test.ts b/apps/cli/test/gitlab-projectors.test.ts index 3a206615c..f3d2685b0 100644 --- a/apps/cli/test/gitlab-projectors.test.ts +++ b/apps/cli/test/gitlab-projectors.test.ts @@ -5,6 +5,7 @@ import { ProjectorRegistry, SignalSourceRegistry, makeEventId, + validateGitLabEventData, validateMapleCloudEvent, type SignalPredicate, type SignalProjectionSpec, @@ -14,10 +15,14 @@ import identities from "./fixtures/gitlab-projector-identities.v1.json" import { gitlabDeploymentLifecycleProjector, gitlabIssueCommentProjector, + gitlabIssueLifecycleProjector, gitlabJobLifecycleProjector, + gitlabMergeRequestCommentProjector, gitlabMergeRequestLifecycleProjector, gitlabPipelineCompletedProjector, gitlabProjectLifecycleProjector, + gitlabRefLifecycleProjector, + gitlabReleaseLifecycleProjector, registerGitLabProjectors, } from "../src/server/eventing/gitlab-projectors" import { OTLP_LOG_ADAPTER, normalizeOtlpLogs } from "../src/server/eventing/otlp" @@ -68,7 +73,7 @@ const gitlabEvent = (eventName: string, eventId: string, extra: readonly unknown }, scopeLogs: [ { - scope: { name: "srvmini2.gitlab.repository-events", version: "1" }, + scope: { name: "example.gitlab.repository-events", version: "1" }, logRecords: [ { timeUnixNano: "1786131720123456789", @@ -78,14 +83,14 @@ const gitlabEvent = (eventName: string, eventId: string, extra: readonly unknown body: { stringValue: `gitlab event ${eventName}` }, attributes: [ stringAttr("event.id", eventId), - stringAttr("event.source", "https://gitlab.internal"), + stringAttr("event.source", "https://gitlab.example.test"), stringAttr("event.name", eventName), stringAttr("gitlab.event.id", eventId), stringAttr("gitlab.event.result", "success"), intAttr("gitlab.project.id", "42"), - stringAttr("gitlab.project.path", "rdev/maple"), + stringAttr("gitlab.project.path", "example/widgets"), intAttr("gitlab.actor.id", "9"), - stringAttr("gitlab.actor.name", "rdev"), + stringAttr("gitlab.actor.name", "maintainer"), ...extra, ], }, @@ -140,7 +145,7 @@ describe("GitLab event projectors", () => { it("projects the production receiver's project lifecycle fields", () => { const signal = normalizedEvent( gitlabEvent("project_rename", "project-event-42", [ - stringAttr("gitlab.project.old_path", "rdev/old-maple"), + stringAttr("gitlab.project.old_path", "example/old-widgets"), ]), ) const result = evaluate( @@ -152,14 +157,14 @@ describe("GitLab event projectors", () => { strictEqual(result.failures.length, 0) strictEqual(result.events[0]?.id, samples[0]?.id) deepStrictEqual(result.events[0]?.data, { - project: { id: "42", path: "rdev/maple", oldPath: "rdev/old-maple" }, + project: { id: "42", path: "example/widgets", oldPath: "example/old-widgets" }, action: "renamed", sourceEvent: "project_rename", - actor: { id: "9", name: "rdev" }, + actor: { id: "9", name: "maintainer" }, result: "success", serviceName: "gitlab-repository-events", }) - strictEqual(result.events[0]?.subject, "rdev/maple") + strictEqual(result.events[0]?.subject, "example/widgets") }) it("maps every normalized project lifecycle event to a version-1 action", () => { @@ -191,6 +196,11 @@ describe("GitLab event projectors", () => { const signal = normalizedEvent( gitlabEvent("merge_request_open", "mr-event-7", [ intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), stringAttr("gitlab.merge_request.source_branch", "feature/maple"), stringAttr("gitlab.merge_request.target_branch", "main"), stringAttr("gitlab.merge_request.commit", "abc123"), @@ -205,20 +215,22 @@ describe("GitLab event projectors", () => { strictEqual(result.failures.length, 0) strictEqual(result.events[0]?.id, samples[1]?.id) deepStrictEqual(result.events[0]?.data, { - project: { id: "42", path: "rdev/maple" }, + project: { id: "42", path: "example/widgets" }, mergeRequest: { iid: "7", + title: "Freeze contracts", + url: "https://gitlab.example.test/example/widgets/-/merge_requests/7", action: "open", sourceBranch: "feature/maple", targetBranch: "main", commit: "abc123", }, sourceEvent: "merge_request_open", - actor: { id: "9", name: "rdev" }, + actor: { id: "9", name: "maintainer" }, result: "success", serviceName: "gitlab-repository-events", }) - strictEqual(result.events[0]?.subject, "rdev/maple/-/merge_requests/7") + strictEqual(result.events[0]?.subject, "example/widgets/-/merge_requests/7") }) it("projects only terminal pipeline events from the normalized receiver fields", () => { @@ -231,6 +243,13 @@ describe("GitLab event projectors", () => { stringAttr("gitlab.ci.pipeline.detailed_status", "failed"), stringAttr("gitlab.ci.pipeline.name", "Maple CI"), stringAttr("gitlab.ci.pipeline.source", "push"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + intAttr("gitlab.ci.pipeline.failed_job_count", "0"), + boolAttr("gitlab.ci.pipeline.failed_jobs_truncated", false), + failedJobsAttr([]), stringAttr("vcs.ref", "main"), stringAttr("vcs.ref.head.revision", "deadbeef"), intAttr("gitlab.ci.pipeline.duration_ms", "63000"), @@ -247,7 +266,7 @@ describe("GitLab event projectors", () => { strictEqual(result.failures.length, 0) strictEqual(result.events[0]?.id, samples[2]?.id) deepStrictEqual(result.events[0]?.data, { - project: { id: "42", path: "rdev/maple" }, + project: { id: "42", path: "example/widgets" }, pipeline: { id: "900", iid: "12", @@ -260,13 +279,17 @@ describe("GitLab event projectors", () => { durationMs: "63000", queuedDurationMs: "10000", stageCount: "3", + url: "https://gitlab.example.test/example/widgets/-/pipelines/900", + failedJobCount: "0", + failedJobsTruncated: false, + failedJobs: [], }, sourceEvent: "ci_pipeline_completed", - actor: { id: "9", name: "rdev" }, + actor: { id: "9", name: "maintainer" }, result: "failure", serviceName: "gitlab-repository-events", }) - strictEqual(result.events[0]?.subject, "rdev/maple/-/pipelines/900") + strictEqual(result.events[0]?.subject, "example/widgets/-/pipelines/900") }) it("projects every issue lifecycle action and sanitized issue comments", () => { @@ -284,8 +307,12 @@ describe("GitLab event projectors", () => { gitlabEvent(sourceEvent, `delivery-issue:${sourceEvent}`, [ intAttr("gitlab.issue.id", "70"), intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), stringAttr("gitlab.issue.title", "Typed events"), - stringAttr("gitlab.issue.url", "https://gitlab.internal/rdev/maple/-/issues/7"), + stringAttr( + "gitlab.issue.url", + "https://gitlab.example.test/example/widgets/-/issues/7", + ), stringArrayAttr("gitlab.issue.labels", ["agent-ready", "backend"]), ]), ), @@ -301,27 +328,48 @@ describe("GitLab event projectors", () => { normalizedEvent( gitlabEvent("issue_comment", "delivery-comment:0", [ intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), intAttr("gitlab.comment.id", "81"), stringArrayAttr("gitlab.issue.labels", ["agent-ready", "backend"]), stringAttr("gitlab.comment.excerpt", " hello\u0000\n\tMaple "), - stringAttr("gitlab.comment.url", "https://gitlab.internal/rdev/maple/-/issues/7#note_81"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), ]), ), ) strictEqual(commentResult.failures.length, 0) deepStrictEqual(commentResult.events[0]?.data, { - project: { id: "42", path: "rdev/maple" }, - issue: { iid: "7", labels: ["agent-ready", "backend"] }, + project: { id: "42", path: "example/widgets" }, + issue: { iid: "7", type: "issue", labels: ["agent-ready", "backend"] }, comment: { id: "81", excerpt: "hello Maple", - url: "https://gitlab.internal/rdev/maple/-/issues/7#note_81", + url: "https://gitlab.example.test/example/widgets/-/issues/7#note_81", }, sourceEvent: "issue_comment", - actor: { id: "9", name: "rdev" }, + actor: { id: "9", name: "maintainer" }, result: "success", serviceName: "gitlab-repository-events", }) + + const workItem = evaluate( + "gitlab.issue.lifecycle", + "gitlab-work-item-updated", + "issue_update", + normalizedEvent( + gitlabEvent("issue_update", "delivery-work-item:0", [ + intAttr("gitlab.issue.iid", "19"), + stringAttr("gitlab.issue.type", "incident_response_task"), + ]), + ), + ) + deepStrictEqual((workItem.events[0]?.data as { issue: { type: string; action: string } }).issue, { + iid: "19", + type: "incident_response_task", + action: "update", + }) }) it("uses an explicit MR lifecycle vocabulary and distinguishes review comments", () => { @@ -338,7 +386,7 @@ describe("GitLab event projectors", () => { stringAttr("gitlab.merge_request.title", "Freeze contracts"), stringAttr( "gitlab.merge_request.url", - "https://gitlab.internal/rdev/maple/-/merge_requests/7", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", ), ...(sourceEvent === "merge_request_review" ? [stringAttr("gitlab.merge_request.review_state", "approved")] @@ -368,8 +416,17 @@ describe("GitLab event projectors", () => { normalizedEvent( gitlabEvent(sourceEvent, `delivery-mr-comment:${sourceEvent}`, [ intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), intAttr("gitlab.comment.id", "82"), stringAttr("gitlab.comment.excerpt", "Please add a fixture"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7#note_82", + ), ]), ), ) @@ -390,7 +447,7 @@ describe("GitLab event projectors", () => { intAttr("gitlab.ci.pipeline.merge_request_iid", "7"), stringAttr( "gitlab.ci.pipeline.url", - "https://gitlab.internal/rdev/maple/-/pipelines/900", + "https://gitlab.example.test/example/widgets/-/pipelines/900", ), intAttr("gitlab.ci.pipeline.failed_job_count", "3"), boolAttr("gitlab.ci.pipeline.failed_jobs_truncated", true), @@ -399,7 +456,7 @@ describe("GitLab event projectors", () => { id: "901", name: "unit", stage: "test", - url: "https://gitlab.internal/rdev/maple/-/jobs/901", + url: "https://gitlab.example.test/example/widgets/-/jobs/901", }, { id: "902", name: "integration", stage: "test" }, ]), @@ -414,7 +471,7 @@ describe("GitLab event projectors", () => { id: "900", mergeRequestIid: "7", status: "failed", - url: "https://gitlab.internal/rdev/maple/-/pipelines/900", + url: "https://gitlab.example.test/example/widgets/-/pipelines/900", failedJobCount: "3", failedJobsTruncated: true, failedJobs: [ @@ -423,7 +480,7 @@ describe("GitLab event projectors", () => { name: "unit", stage: "test", status: "failed", - url: "https://gitlab.internal/rdev/maple/-/jobs/901", + url: "https://gitlab.example.test/example/widgets/-/jobs/901", }, { id: "902", name: "integration", stage: "test", status: "failed" }, ], @@ -452,7 +509,7 @@ describe("GitLab event projectors", () => { stringAttr("gitlab.deployment.revision", "a8123fa"), stringAttr( "gitlab.deployment.url", - "https://gitlab.internal/rdev/maple/-/deployments/501", + "https://gitlab.example.test/example/widgets/-/deployments/501", ), ]), ), @@ -544,6 +601,135 @@ describe("GitLab event projectors", () => { } }) + it("requires source-provided identity and remains deterministic across retries for every new family", () => { + const mr = [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), + ] + const cases = [ + { + name: "project", + projector: gitlabProjectLifecycleProjector, + request: gitlabEvent("project_update", "retry-project"), + }, + { + name: "issue", + projector: gitlabIssueLifecycleProjector, + request: gitlabEvent("issue_update", "retry-issue", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "task"), + ]), + }, + { + name: "issue comment", + projector: gitlabIssueCommentProjector, + request: gitlabEvent("issue_comment", "retry-issue-comment", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "task"), + intAttr("gitlab.comment.id", "81"), + stringAttr("gitlab.comment.excerpt", "Bounded comment"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), + ]), + }, + { + name: "merge request", + projector: gitlabMergeRequestLifecycleProjector, + request: gitlabEvent("merge_request_update", "retry-mr", mr), + }, + { + name: "merge request comment", + projector: gitlabMergeRequestCommentProjector, + request: gitlabEvent("merge_request_comment", "retry-mr-comment", [ + ...mr, + intAttr("gitlab.comment.id", "82"), + stringAttr("gitlab.comment.excerpt", "Bounded review"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7#note_82", + ), + ]), + }, + { + name: "pipeline", + projector: gitlabPipelineCompletedProjector, + request: gitlabEvent("ci_pipeline_completed", "retry-pipeline", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "success"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + ]), + }, + { + name: "deployment", + projector: gitlabDeploymentLifecycleProjector, + request: gitlabEvent("deployment_success", "retry-deployment", [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "staging"), + stringAttr("gitlab.deployment.status", "success"), + stringAttr("gitlab.deployment.revision", "a8123fa"), + stringAttr( + "gitlab.deployment.url", + "https://gitlab.example.test/example/widgets/-/deployments/501", + ), + ]), + }, + { + name: "job", + projector: gitlabJobLifecycleProjector, + request: gitlabEvent("ci_job_success", "retry-job", [ + intAttr("gitlab.ci.job.id", "901"), + stringAttr("gitlab.ci.job.name", "unit"), + stringAttr("gitlab.ci.job.status", "success"), + ]), + }, + { + name: "ref", + projector: gitlabRefLifecycleProjector, + request: gitlabEvent("branch_update", "retry-ref", [ + stringAttr("vcs.ref", "refs/heads/main"), + stringAttr("vcs.ref.base.revision", "a8123fa"), + stringAttr("vcs.ref.head.revision", "b9123fa"), + ]), + }, + { + name: "release", + projector: gitlabReleaseLifecycleProjector, + request: gitlabEvent("release_update", "retry-release", [ + stringAttr("gitlab.release.tag", "v1.0.0"), + ]), + }, + ] as const + + for (const testCase of cases) { + const first = normalizedEvent(testCase.request) + const retry = normalizedEvent(testCase.request) + deepStrictEqual( + testCase.projector.project(first), + testCase.projector.project(retry), + testCase.name, + ) + throws( + () => + testCase.projector.project({ + ...first, + occurrenceId: "derived:sha256:fixture", + identityQuality: "derived", + }), + /source-provided occurrence identity/, + testCase.name, + ) + } + }) + it("keeps source identity and the CloudEvent ID deterministic across retries", () => { const first = normalizedEvent(gitlabEvent("project_create", "project-event-42")) const retry = normalizedEvent(gitlabEvent("project_create", "project-event-42")) @@ -564,7 +750,7 @@ describe("GitLab event projectors", () => { strictEqual(firstResult.events[0]?.id, retryResult.events[0]?.id) strictEqual( firstResult.events[0]?.id, - "sha256:8f45527a4e615e057142026df477f20e2e6d63ae55bc761ce4dc151dda706811", + "sha256:58a73bebd375da87aef9ce33d28376f127200a83d15d376a1d1d872d3757d653", ) strictEqual(firstResult.events[0]?.id?.startsWith("sha256:"), true) }) @@ -596,6 +782,95 @@ describe("GitLab event projectors", () => { strictEqual(badIdResult.failures[0]?.message, "GitLab projector gitlab.project.id must be an int64") }) + it("requires every semantically available producer handoff field", () => { + const issueWithoutType = normalizedEvent( + gitlabEvent("issue_update", "issue-no-type", [intAttr("gitlab.issue.iid", "7")]), + ) + throws(() => gitlabIssueLifecycleProjector.project(issueWithoutType), /missing gitlab.issue.type/) + + const malformedWorkItemType = normalizedEvent( + gitlabEvent("issue_update", "issue-bad-type", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "Incident Response Task"), + ]), + ) + throws(() => gitlabIssueLifecycleProjector.project(malformedWorkItemType), /lowercase token/) + + const commentWithoutExcerpt = normalizedEvent( + gitlabEvent("issue_comment", "comment-no-excerpt", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), + intAttr("gitlab.comment.id", "81"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), + ]), + ) + throws( + () => gitlabIssueCommentProjector.project(commentWithoutExcerpt), + /missing gitlab.comment.excerpt/, + ) + + const mergeRequestWithoutTitle = normalizedEvent( + gitlabEvent("merge_request_open", "mr-no-title", [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), + ]), + ) + throws( + () => gitlabMergeRequestLifecycleProjector.project(mergeRequestWithoutTitle), + /missing gitlab.merge_request.title/, + ) + + const pipelineWithoutUrl = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-no-url", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "success"), + ]), + ) + throws( + () => gitlabPipelineCompletedProjector.project(pipelineWithoutUrl), + /missing gitlab.ci.pipeline.url/, + ) + + const failedPipelineWithoutSummaryCount = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-no-summary-count", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + boolAttr("gitlab.ci.pipeline.failed_jobs_truncated", false), + failedJobsAttr([]), + ]), + ) + throws( + () => gitlabPipelineCompletedProjector.project(failedPipelineWithoutSummaryCount), + /missing gitlab.ci.pipeline.failed_job_count/, + ) + + const deploymentWithoutRevision = normalizedEvent( + gitlabEvent("deployment_success", "deployment-no-revision", [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "production"), + stringAttr("gitlab.deployment.status", "success"), + stringAttr( + "gitlab.deployment.url", + "https://gitlab.example.test/example/widgets/-/deployments/501", + ), + ]), + ) + throws( + () => gitlabDeploymentLifecycleProjector.project(deploymentWithoutRevision), + /missing gitlab.deployment.revision/, + ) + }) + it("rejects nonterminal pipeline status, unknown actions, and unknown config fields", () => { const running = normalizedEvent( gitlabEvent("ci_pipeline_completed", "pipeline-running", [ @@ -654,8 +929,10 @@ describe("GitLab event projectors", () => { const unsafeUrl = normalizedEvent( gitlabEvent("issue_comment", "comment-query", [ intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), intAttr("gitlab.comment.id", "81"), - stringAttr("gitlab.comment.url", "https://gitlab.internal/note?private=value"), + stringAttr("gitlab.comment.excerpt", "Safe bounded excerpt"), + stringAttr("gitlab.comment.url", "https://gitlab.example.test/note?private=value"), ]), ) throws(() => gitlabIssueCommentProjector.project(unsafeUrl), /must not contain a query or fragment/) @@ -664,6 +941,10 @@ describe("GitLab event projectors", () => { gitlabEvent("ci_pipeline_completed", "pipeline-too-many-jobs", [ intAttr("gitlab.ci.pipeline.id", "900"), stringAttr("gitlab.ci.pipeline.status", "failed"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), failedJobsAttr( Array.from({ length: 21 }, (_, index) => ({ id: String(index + 1), @@ -677,8 +958,13 @@ describe("GitLab event projectors", () => { const oversizedExcerpt = normalizedEvent( gitlabEvent("issue_comment", "comment-too-long", [ intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), intAttr("gitlab.comment.id", "81"), stringAttr("gitlab.comment.excerpt", "x".repeat(1025)), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), ]), ) throws(() => gitlabIssueCommentProjector.project(oversizedExcerpt), /exceeds 1024 UTF-8 bytes/) @@ -686,6 +972,7 @@ describe("GitLab event projectors", () => { const tooManyLabels = normalizedEvent( gitlabEvent("issue_open", "issue-too-many-labels", [ intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), stringArrayAttr( "gitlab.issue.labels", Array.from({ length: 51 }, (_, index) => `label-${index}`), @@ -739,12 +1026,13 @@ describe("GitLab event projectors", () => { strictEqual(identities.length, samples.length) for (const [index, sample] of samples.entries()) { strictEqual(validateMapleCloudEvent(sample).event.id, sample.id) + validateGitLabEventData(sample.dataschema, sample.data) const identity = identities[index]! strictEqual( makeEventId({ tenantId: "local", sourceKind: "otel.log", - source: "https://gitlab.internal", + source: "https://gitlab.example.test", occurrenceId: identity.occurrenceId, projectionId: identity.projectionId, projectionRevision: 1, diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts index 8db98d366..c6c10ebda 100644 --- a/apps/cli/test/local-eventing-control-store.test.ts +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path" import { describe, it } from "vitest" import type { MapleCloudEvent, SignalProjectionSpec } from "@maple/eventing-core" import { eventingControlPath, LocalEventingControlStore } from "../src/server/eventing/control-store" +import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-eventing-control-")) @@ -53,6 +54,75 @@ const event = (overrides: Partial = {}): MapleCloudEvent => ({ }) describe("LocalEventingControlStore", () => { + it("records bounded outbox and consumer telemetry without identifiers or payloads", async () => + withDataDir(async (dataDir) => { + const observations: EventingTelemetryObservation[] = [] + const store = await LocalEventingControlStore.open(dataDir, undefined, { + record: (observation) => observations.push(observation), + }) + try { + const sensitiveEvent = event({ + data: { iid: 42, title: "PAYLOAD-MUST-NOT-BE-METRIC-DATA" }, + }) + store.stageEvents([sensitiveEvent, sensitiveEvent]) + throws(() => store.stageEvents([event({ data: { iid: 43 } })]), /collision/) + throws(() => store.markReady(["unknown-event-identifier"]), /unknown event/) + store.markReady([sensitiveEvent.id]) + store.registerConsumer("tenant-a", "private-consumer-identifier", "beginning") + const claim = store.claimReady("tenant-a", "private-consumer-identifier", 10, 30) + throws( + () => store.claimReady("tenant-a", "private-consumer-identifier", 10, 30), + /active lease/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + "incorrect-private-token", + claim.throughSequence!, + ), + /token does not match/, + ) + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + claim.leaseToken!, + claim.throughSequence!, + ) + + const operationOutcomes = observations.map( + ({ operation, outcome }) => `${operation}:${outcome}`, + ) + for (const expected of [ + "outbox_stage:success", + "outbox_stage:failure", + "outbox_ready:success", + "outbox_ready:failure", + "outbox_dedup:success", + "consumer_claim:success", + "consumer_claim:failure", + "consumer_ack:success", + "consumer_ack:failure", + "consumer_lease:failure", + "consumer_lag:observed", + ]) + ok(operationOutcomes.includes(expected), `missing telemetry observation ${expected}`) + + const serialized = JSON.stringify(observations) + for (const forbidden of [ + "PAYLOAD-MUST-NOT-BE-METRIC-DATA", + "private-consumer-identifier", + "incorrect-private-token", + sensitiveEvent.id, + claim.leaseToken!, + ]) + strictEqual(serialized.includes(forbidden), false) + } finally { + store.close() + } + })) + it("stores immutable sequential revisions and only loads the active revision", async () => withDataDir(async (dataDir) => { const store = await LocalEventingControlStore.open(dataDir) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts index a415ffefe..f9ea5300b 100644 --- a/apps/cli/test/local-eventing-runtime.test.ts +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -1,4 +1,4 @@ -import { deepStrictEqual, strictEqual, throws } from "node:assert" +import { deepStrictEqual, ok, strictEqual, throws } from "node:assert" import { mkdirSync, mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" @@ -7,6 +7,7 @@ import type { SignalProjectionSpec } from "@maple/eventing-core" import { LocalEventingControlStore } from "../src/server/eventing/control-store" import { normalizeOtlpLogs } from "../src/server/eventing/otlp" import { LocalEventingRuntime } from "../src/server/eventing/runtime" +import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" import { encodeLogs } from "../src/server/otlp/encode" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { @@ -44,14 +45,14 @@ const gitlabIssueCreated = { body: { stringValue: "Issue 42 created" }, attributes: [ attr("event.id", { stringValue: "01K20GITLABISSUE42" }), - attr("event.source", { stringValue: "https://gitlab.internal" }), + attr("event.source", { stringValue: "https://gitlab.example.test" }), attr("gitlab.project.id", { intValue: "7" }), - attr("gitlab.project.path", { stringValue: "platform/maple" }), + attr("gitlab.project.path", { stringValue: "example/widgets" }), attr("gitlab.issue.id", { intValue: "4200" }), attr("gitlab.issue.iid", { intValue: "42" }), attr("gitlab.issue.title", { stringValue: "Wire GitLab events" }), attr("gitlab.issue.url", { - stringValue: "https://gitlab.internal/platform/maple/-/issues/42", + stringValue: "https://gitlab.example.test/example/widgets/-/issues/42", }), attr("gitlab.user.id", { intValue: "9" }), attr("gitlab.user.username", { stringValue: "operator" }), @@ -94,11 +95,44 @@ const projection = (overrides: Partial = {}): SignalProjec }) describe("LocalEventingRuntime", () => { + it("records bounded normalization and projection outcomes without signal data", async () => + withDataDir(async (dataDir) => { + const observations: EventingTelemetryObservation[] = [] + const telemetry = { + record: (observation: EventingTelemetryObservation) => observations.push(observation), + } + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, telemetry) + runtime.activate(projection()) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 1) + + const malformed = structuredClone(gitlabIssueCreated) + firstLogRecord(malformed).attributes = firstLogRecord(malformed).attributes.filter( + ({ key }) => key !== "gitlab.project.path", + ) + strictEqual(runtime.evaluateOtlp("logs", malformed).failures.length, 1) + + const operationOutcomes = observations.map( + ({ operation, outcome }) => `${operation}:${outcome}`, + ) + ok(operationOutcomes.includes("normalization:success")) + ok(operationOutcomes.includes("projection:success")) + ok(operationOutcomes.includes("projection:failure")) + const serialized = JSON.stringify(observations) + strictEqual(serialized.includes("Wire GitLab events"), false) + strictEqual(serialized.includes("01K20GITLABISSUE42"), false) + strictEqual(serialized.includes("gitlab-issue-created"), false) + } finally { + store.close() + } + })) + it("normalizes typed GitLab OTLP fields while preserving the existing warehouse encoding", () => { const [signal] = normalizeOtlpLogs(gitlabIssueCreated, "2026-08-07T20:00:00Z") strictEqual(signal?.occurrenceId, "01K20GITLABISSUE42") strictEqual(signal?.identityQuality, "source") - strictEqual(signal?.source, "https://gitlab.internal") + strictEqual(signal?.source, "https://gitlab.example.test") deepStrictEqual(signal?.fields.get("attribute:gitlab.issue.iid"), { type: "int64", value: "42", @@ -185,9 +219,9 @@ describe("LocalEventingRuntime", () => { deepStrictEqual(first.events[0], { specversion: "1.0", id: first.events[0]!.id, - source: "https://gitlab.internal", + source: "https://gitlab.example.test", type: "dev.maple.gitlab.issue.created.v1", - subject: "platform/maple/issues/42", + subject: "example/widgets/issues/42", time: "2026-08-07T19:42:00.123456789Z", datacontenttype: "application/json", dataschema: "urn:maple:event-schema:gitlab-issue-created:v1", @@ -199,12 +233,12 @@ describe("LocalEventingRuntime", () => { sourceoccurrenceid: "01K20GITLABISSUE42", sourceidentityquality: "source", data: { - project: { id: "7", path: "platform/maple" }, + project: { id: "7", path: "example/widgets" }, issue: { id: "4200", iid: "42", title: "Wire GitLab events", - url: "https://gitlab.internal/platform/maple/-/issues/42", + url: "https://gitlab.example.test/example/widgets/-/issues/42", }, actor: { id: "9", username: "operator" }, serviceName: "gitlab-rails", diff --git a/apps/cli/test/server-args.test.ts b/apps/cli/test/server-args.test.ts index b86f1d937..0021679c8 100644 --- a/apps/cli/test/server-args.test.ts +++ b/apps/cli/test/server-args.test.ts @@ -30,7 +30,10 @@ describe("local server bind host", () => { it("separates the bind address from the client-facing address", () => { strictEqual(resolveAdvertiseHost(undefined, undefined, "0.0.0.0"), "127.0.0.1") - strictEqual(resolveAdvertiseHost(undefined, " srvmini2.lan ", "0.0.0.0"), "srvmini2.lan") + strictEqual( + resolveAdvertiseHost(undefined, " node-a.example.test ", "0.0.0.0"), + "node-a.example.test", + ) strictEqual(resolveAdvertiseHost(" 192.0.2.10 ", "ignored", "0.0.0.0"), "192.0.2.10") strictEqual(resolveAdvertiseHost(" ", " [::1] ", "0.0.0.0"), "::1") }) @@ -70,7 +73,7 @@ describe("buildDetachedChildArgs", () => { const args = buildDetachedChildArgs({ entry: "/repo/apps/cli/src/bin.ts", host: "0.0.0.0", - advertiseHost: "srvmini2.lan", + advertiseHost: "node-a.example.test", port: 4318, dataDir: "/tmp/maple data", offline: true, @@ -84,7 +87,7 @@ describe("buildDetachedChildArgs", () => { "--host", "0.0.0.0", "--advertise-host", - "srvmini2.lan", + "node-a.example.test", "--port", "4318", "--data-dir", diff --git a/apps/cli/test/server-network.test.ts b/apps/cli/test/server-network.test.ts index 862f010c2..d9481e8b0 100644 --- a/apps/cli/test/server-network.test.ts +++ b/apps/cli/test/server-network.test.ts @@ -100,18 +100,23 @@ describe("local listener addresses", () => { }) describe("browser origin policy", () => { - const requestUrl = new URL("http://srvmini2.lan:4418/local/query") + const requestUrl = new URL("http://node-a.example.test:4418/local/query") const hostedOrigin = "https://local.maple.dev" - const browserHosts = ["srvmini2.lan", "127.0.0.1"] + const browserHosts = ["node-a.example.test", "127.0.0.1"] it("allows non-browser clients, the advertised same-origin UI, and the hosted UI", () => { strictEqual(isBrowserOriginAllowed(requestUrl, null, hostedOrigin, browserHosts), true) strictEqual( - isBrowserOriginAllowed(requestUrl, "http://srvmini2.lan:4418", hostedOrigin, browserHosts), + isBrowserOriginAllowed(requestUrl, "http://node-a.example.test:4418", hostedOrigin, browserHosts), true, ) strictEqual( - isBrowserOriginAllowed(requestUrl, "https://srvmini2.lan:4418", hostedOrigin, browserHosts), + isBrowserOriginAllowed( + requestUrl, + "https://node-a.example.test:4418", + hostedOrigin, + browserHosts, + ), true, ) strictEqual(isBrowserOriginAllowed(requestUrl, hostedOrigin, hostedOrigin, browserHosts), true) diff --git a/apps/local-ui/src/lib/constants.test.ts b/apps/local-ui/src/lib/constants.test.ts index cf150b2a5..54ad10873 100644 --- a/apps/local-ui/src/lib/constants.test.ts +++ b/apps/local-ui/src/lib/constants.test.ts @@ -19,9 +19,9 @@ describe("local UI endpoint selection", () => { }) it("keeps an embedded LAN or TLS-proxied UI same-origin", () => { - const page = location("https://srvmini2.lan:4418/?api_key=not-propagated") + const page = location("https://node-a.example.test:4418/?api_key=not-propagated") expect(localApiBaseForLocation(page)).toBe("") - expect(localOtlpEndpointForLocation(page)).toBe("https://srvmini2.lan:4418") + expect(localOtlpEndpointForLocation(page)).toBe("https://node-a.example.test:4418") }) it("keeps the Vite development UI same-origin for its proxied query and OTLP routes", () => { diff --git a/docs/gitlab-event-projectors.md b/docs/gitlab-event-projectors.md index 9a8b5c99a..34f9399ed 100644 --- a/docs/gitlab-event-projectors.md +++ b/docs/gitlab-event-projectors.md @@ -17,6 +17,13 @@ occurrence ID and Maple derives the CloudEvent ID from tenant, source kind, source, occurrence ID, projection ID, and projection revision. Payload hashes are audit data, not identity. +Every new GitLab projector fails closed unless the normalized occurrence has +`sourceidentityquality: "source"` and a non-null source occurrence ID. This +prevents retries of one GitLab delivery from becoming distinct events. The +generic eventing adapter still supports deterministic derived identity for +other projector families; the restriction is intentionally at the new GitLab +projector boundary. + New envelopes expose that preserved input as optional, backward-compatible CloudEvents extensions `sourceoccurrenceid` and `sourceidentityquality`. Historical envelopes lacking both fields remain schema-valid. A downstream @@ -35,11 +42,17 @@ Complete output fixtures live in projection identity inputs are paired by index in `apps/cli/test/fixtures/gitlab-projector-identities.v1.json`. +Each factual GitLab `data` payload also has a checked-in JSON Schema under +`packages/eventing-core/schemas`. Fixture tests decode against the schema +selected by `dataschema`, and `schemas:check` fails when generated schemas +drift from the checked-in files. + ## Common input and safety contract Every new projector requires: - `event.name`: one explicit event name documented below; +- source-provided occurrence identity as described above; - `gitlab.project.id`: positive OTLP int64; - `gitlab.project.path`: non-blank bounded text. @@ -104,11 +117,22 @@ Schema: `urn:maple:event-schema:gitlab-issue-lifecycle:v1` | `issue_close` | `close` | | `issue_reopen` | `reopen` | -Required: `gitlab.issue.iid`. Optional: positive `gitlab.issue.id`, -`gitlab.issue.title`, canonical `gitlab.issue.url`, and bounded lowercase -`gitlab.issue.state`. Structured `gitlab.issue.labels` is an optional array of -at most 50 non-blank labels, each at most 256 UTF-8 bytes. The same issue object, -including labels, is used by issue-comment events. +Required: `gitlab.issue.iid` and `gitlab.issue.type`. The type is derived from +Work Item Hook `object_attributes.type`, normalized to a lowercase snake token, +and bounded to 64 UTF-8 bytes. It is deliberately not an enum because GitLab +work-item types are configurable. Optional fields are positive +`gitlab.issue.id`, title, canonical URL, and bounded lowercase state. +Structured `gitlab.issue.labels` is an optional array of at most 50 non-blank +labels, each at most 256 UTF-8 bytes. The same issue object, including required +type and any labels, is used by issue-comment events. + +Project-scoped Work Item Hooks are normalized into this existing family: +create/open, update, close, and reopen facts use `issue_open`, `issue_update`, +`issue_close`, and `issue_reopen`; work-item notes use `issue_comment`. The +producer must place the normalized Work Item type in `gitlab.issue.type` and +must not expose the raw hook payload. Epics require group hooks and are outside +the current user-namespace project-hook coverage; this is an explicit external +coverage limit, not a second Maple event family. ### Issue comments @@ -119,9 +143,10 @@ Type: `dev.maple.gitlab.issue.comment.v1` Schema: `urn:maple:event-schema:gitlab-issue-comment:v1` The only accepted event is `issue_comment`. Required fields are -`gitlab.issue.iid` and positive `gitlab.comment.id`. Optional fields are -`gitlab.comment.excerpt`, canonical `gitlab.comment.url`, and boolean -`gitlab.comment.system`. The full comment body is never projected. +`gitlab.issue.iid`, `gitlab.issue.type`, positive `gitlab.comment.id`, a +sanitized `gitlab.comment.excerpt`, and canonical `gitlab.comment.url`. +`gitlab.comment.system` remains optional because GitLab does not identify every +note source as a system note. The full comment body is never projected. ### Merge-request lifecycle @@ -140,10 +165,11 @@ Schema: `urn:maple:event-schema:gitlab-merge-request-lifecycle:v1` | `merge_request_merge` | `merge` | | `merge_request_review` | `review` | -Required: positive `gitlab.merge_request.iid`. Optional: -`gitlab.merge_request.title`, canonical `gitlab.merge_request.url`, source and -target branches, hexadecimal `gitlab.merge_request.commit`, and bounded -`gitlab.merge_request.review_state`. Review events require `review_state`. +Required: positive `gitlab.merge_request.iid`, bounded +`gitlab.merge_request.title`, and canonical `gitlab.merge_request.url`. +Optional fields are source and target branches, hexadecimal +`gitlab.merge_request.commit`, and bounded `gitlab.merge_request.review_state`. +Review events require `review_state`. Unlike the initial implementation, arbitrary `merge_request_` events fail closed. @@ -157,8 +183,9 @@ Schema: `urn:maple:event-schema:gitlab-merge-request-comment:v1` `merge_request_comment` emits `comment.kind: "comment"` and `merge_request_review_comment` emits `comment.kind: "review"`. Required fields -are `gitlab.merge_request.iid` and `gitlab.comment.id`; the same bounded comment -fields and sanitization rules as issue comments apply. +are `gitlab.merge_request.iid`, MR title and canonical URL, +`gitlab.comment.id`, sanitized excerpt, and canonical comment URL. The same +bounded comment sanitization rules as issue comments apply. ### Completed pipelines @@ -169,30 +196,31 @@ Type: `dev.maple.gitlab.pipeline.completed.v1` Schema: `urn:maple:event-schema:gitlab-pipeline-completed:v1` The only accepted event is `ci_pipeline_completed`. Required fields are -positive `gitlab.ci.pipeline.id` and terminal -`gitlab.ci.pipeline.status: success|failed|canceled|skipped`. Optional scalar -fields are: +positive `gitlab.ci.pipeline.id`, terminal +`gitlab.ci.pipeline.status: success|failed|canceled|skipped`, and canonical +`gitlab.ci.pipeline.url`. Optional scalar fields are: - `gitlab.ci.pipeline.iid`, `name`, `source`, and `detailed_status`; - positive `gitlab.ci.pipeline.merge_request_iid`, emitted only when the pipeline-to-MR association is unambiguous; -- canonical `gitlab.ci.pipeline.url`; - `vcs.ref` and hexadecimal `vcs.ref.head.revision`; - non-negative `duration_ms`, `queued_duration_ms`, and `stage_count`; -- paired `failed_job_count` and boolean `failed_jobs_truncated`. +- for failed pipelines only, required non-negative `failed_job_count`, boolean + `failed_jobs_truncated`, and `failed_jobs` summary array. The structured OTLP attribute `gitlab.ci.pipeline.failed_jobs` is an array of at most 20 objects. Each object permits only positive string-encoded int64 `id`, bounded `name`, optional `stage`, literal status `failed`, and optional -canonical `url`. When summaries are present, count and truncation metadata are -required and must agree with the array length. +canonical `url`. For failed pipelines, count and truncation metadata must agree +with the array length. Non-failed terminal pipelines reject failed-job metadata +rather than presenting a misleading empty failure summary. ```json { "pipeline": { "id": "900", "status": "failed", - "url": "https://gitlab.internal/rdev/maple/-/pipelines/900", + "url": "https://gitlab.example.test/example/widgets/-/pipelines/900", "failedJobCount": "3", "failedJobsTruncated": true, "failedJobs": [ @@ -201,7 +229,7 @@ required and must agree with the array length. "name": "unit", "stage": "test", "status": "failed", - "url": "https://gitlab.internal/rdev/maple/-/jobs/901" + "url": "https://gitlab.example.test/example/widgets/-/jobs/901" } ] } @@ -226,9 +254,9 @@ Schema: `urn:maple:event-schema:gitlab-deployment-lifecycle:v1` | `deployment_manual` | `manual` | Positive `gitlab.deployment.id`, bounded -`gitlab.deployment.environment`, and matching status are required. Optional -fields are hexadecimal `gitlab.deployment.revision` and canonical -`gitlab.deployment.url`. +`gitlab.deployment.environment`, matching status, hexadecimal +`gitlab.deployment.revision`, and canonical `gitlab.deployment.url` are all +required. ### Job lifecycle @@ -268,7 +296,9 @@ The production receiver normalizes raw `push`, `tag_push`, and Required fields are bounded `vcs.ref` and hexadecimal `vcs.ref.base.revision`/`vcs.ref.head.revision`. All-zero revisions are valid. Canonical `gitlab.ref.url` and non-negative `gitlab.push.commit_count` are -optional. Raw delivery names are not accepted by this projector. +optional because delete facts may not have a resolvable web URL and some hook +variants omit a commit count. Raw delivery names are not accepted by this +projector. ### Release lifecycle @@ -280,9 +310,19 @@ Schema: `urn:maple:event-schema:gitlab-release-lifecycle:v1` `release_create`, `release_update`, and `release_delete` map to matching `release.action` values. Bounded `gitlab.release.tag` is required. Positive -`gitlab.release.id`, bounded name, and canonical URL are optional. Release +`gitlab.release.id`, bounded name, and canonical URL are optional because +delete hooks and older GitLab payload variants can omit them. Release descriptions are intentionally excluded. +Other optional fields follow the same source-fidelity rule: actor metadata can +be absent for system actions; issue global ID/title/state/URL and MR branch, +commit, or non-review review-state fields can be absent from some update or +delete hook forms; pipeline timing, ref, detailed status, and MR association can +be absent or ambiguous; failed-job `stage` and `url` can be absent from the +bounded lookup result; job pipeline/ref/timing/URL fields can be absent during +early lifecycle states. Projectors never synthesize these values. Tests cover +minimal valid events and reject omission of every field declared required. + ## Complete sample CloudEvent The checked-in fixture set contains one complete envelope for each output @@ -291,10 +331,10 @@ family and enriched variants where useful. For example: ```json { "specversion": "1.0", - "id": "sha256:b7b29d2a12f061dc3e8b6c3bcb550302783b834541b6a8db7b104b4d14f41464", - "source": "https://gitlab.internal", + "id": "sha256:7f8aea4bc391d3b954fecd9ef5356b02aaeb4289ddfa25ef9c3c16217b0d77d1", + "source": "https://gitlab.example.test", "type": "dev.maple.gitlab.deployment.lifecycle.v1", - "subject": "rdev/maple/-/deployments/501", + "subject": "example/widgets/-/deployments/501", "time": "2026-08-07T19:42:00.123456789Z", "datacontenttype": "application/json", "dataschema": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", @@ -306,13 +346,13 @@ family and enriched variants where useful. For example: "sourceoccurrenceid": "delivery-deployment:deployment_failed", "sourceidentityquality": "source", "data": { - "project": { "id": "42", "path": "rdev/maple" }, + "project": { "id": "42", "path": "example/widgets" }, "deployment": { "id": "501", "environment": "production", "status": "failed", "revision": "a8123fa", - "url": "https://gitlab.internal/rdev/maple/-/deployments/501" + "url": "https://gitlab.example.test/example/widgets/-/deployments/501" }, "sourceEvent": "deployment_failed", "result": "failure", @@ -331,10 +371,16 @@ not belong in these projectors. The receiver must emit the explicit event vocabulary and fields above before a corresponding projection is activated. In particular, a producer upgrade is needed wherever the current receiver does not yet emit issue/comment facts, -MR title/URL/review/comment facts, pipeline canonical URL and bounded failed -jobs, deployment identity/environment/status/revision/URL, job lifecycle -fields, semantic ref transitions, or release facts. The receiver owns webhook +Work Item `object_attributes.type`, MR title/URL/review/comment facts, pipeline +canonical URL and bounded failed-job count/truncation summaries, deployment +identity/environment/status/revision/URL, job lifecycle fields, semantic ref +transitions, or release facts. For project-scoped Work Item Hooks the exact +handoff is: normalize `object_attributes.type` to lowercase snake case in +`gitlab.issue.type`; select one existing `issue_*` event name; emit the same +bounded issue/comment fields as an Issue Hook; and assign the indexed stable +source occurrence ID before OTLP export. The receiver owns webhook normalization, duplicate semantic-transition suppression, source UUID indexing, -excerpt pre-sanitization, and failed-job lookup/truncation. Maple validates and -projects those facts but does not reconstruct missing webhook data or call -GitLab. +excerpt pre-sanitization, canonical URL construction, and failed-job +lookup/truncation. Maple validates and projects those facts but does not +reconstruct missing webhook data or call GitLab. Group-hook Epic coverage must +be implemented by the producer if that external scope is later authorized. diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md index 208a5464f..3a2a73a15 100644 --- a/docs/signal-to-event-projection.md +++ b/docs/signal-to-event-projection.md @@ -723,6 +723,16 @@ Required low-cardinality telemetry includes: Raw field values, subjects, event IDs, and arbitrary event types must not become unbounded metric labels. +Maple Local implements the ingest-time subset as +`maple.eventing.operations_total`, `maple.eventing.operation_duration_ms`, and +`maple.eventing.consumer_lag_events`. Their only attributes are bounded +`operation`, `outcome`, and `source_kind` values. The operations cover +normalization, projection success/failure, outbox stage/ready/deduplication, and +consumer claim/ack/lease/lag. Tenant IDs, projection and consumer IDs, event +types and IDs, URLs, payload fields, lease tokens, and credentials are never +metric attributes. Replay and stranded-outbox telemetry remain applicable only +when those optional host operations run. + ## Compatibility and migration This design extends rather than replaces the host-neutral alert-core extraction diff --git a/packages/eventing-core/schemas/gitlab-deployment-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-deployment-lifecycle.v1.schema.json new file mode 100644 index 000000000..b6b3a048d --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-deployment-lifecycle.v1.schema.json @@ -0,0 +1,190 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", + "$ref": "#/$defs/GitLabDeploymentLifecycleDataV1", + "$defs": { + "GitLabDeploymentLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "deployment": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "environment": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "status": { + "type": "string", + "enum": ["running", "success", "failed", "canceled", "blocked", "manual"] + }, + "revision": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + } + }, + "required": ["id", "environment", "status", "revision", "url"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "deployment", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-issue-comment.v1.schema.json b/packages/eventing-core/schemas/gitlab-issue-comment.v1.schema.json new file mode 100644 index 000000000..ccfd79aad --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-issue-comment.v1.schema.json @@ -0,0 +1,280 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-issue-comment:v1", + "$ref": "#/$defs/GitLabIssueCommentDataV1", + "$defs": { + "GitLabIssueCommentDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "issue": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "state": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + } + ] + }, + "allOf": [ + { + "maxItems": 50 + } + ] + } + }, + "required": ["iid", "type"], + "additionalProperties": false + }, + "comment": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "kind": { + "type": "string", + "enum": ["comment", "review"] + }, + "excerpt": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 1024 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?(?:#[^?#\\s]+)?$" + } + ] + }, + "system": { + "type": "boolean" + } + }, + "required": ["id", "excerpt", "url"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "issue", "comment", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-issue-created.v1.schema.json b/packages/eventing-core/schemas/gitlab-issue-created.v1.schema.json new file mode 100644 index 000000000..08f996297 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-issue-created.v1.schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-issue-created:v1", + "$ref": "#/$defs/GitLabIssueCreatedDataV1", + "$defs": { + "GitLabIssueCreatedDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "path": { + "type": "string" + } + }, + "required": ["path"], + "additionalProperties": false + }, + "issue": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "iid": { + "type": "string" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": ["iid"], + "additionalProperties": false + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "additionalProperties": false + }, + "serviceName": { + "type": "string" + }, + "body": {} + }, + "required": ["project", "issue"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-issue-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-issue-lifecycle.v1.schema.json new file mode 100644 index 000000000..9aebaee8b --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-issue-lifecycle.v1.schema.json @@ -0,0 +1,234 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-issue-lifecycle:v1", + "$ref": "#/$defs/GitLabIssueLifecycleDataV1", + "$defs": { + "GitLabIssueLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "issue": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "state": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + } + ] + }, + "allOf": [ + { + "maxItems": 50 + } + ] + }, + "action": { + "type": "string", + "enum": ["open", "update", "close", "reopen"] + } + }, + "required": ["iid", "type", "action"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "issue", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-job-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-job-lifecycle.v1.schema.json new file mode 100644 index 000000000..21f1edcfa --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-job-lifecycle.v1.schema.json @@ -0,0 +1,249 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-job-lifecycle:v1", + "$ref": "#/$defs/GitLabJobLifecycleDataV1", + "$defs": { + "GitLabJobLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "job": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "status": { + "type": "string", + "enum": [ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled" + ] + }, + "stage": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "pipelineId": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "ref": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "revision": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "durationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "allowFailure": { + "type": "boolean" + } + }, + "required": ["id", "name", "status"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "job", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-merge-request-comment.v1.schema.json b/packages/eventing-core/schemas/gitlab-merge-request-comment.v1.schema.json new file mode 100644 index 000000000..390003681 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-merge-request-comment.v1.schema.json @@ -0,0 +1,272 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-merge-request-comment:v1", + "$ref": "#/$defs/GitLabMergeRequestCommentDataV1", + "$defs": { + "GitLabMergeRequestCommentDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "mergeRequest": { + "type": "object", + "properties": { + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "sourceBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "targetBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "commit": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "reviewState": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + } + }, + "required": ["iid", "title", "url"], + "additionalProperties": false + }, + "comment": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "kind": { + "type": "string", + "enum": ["comment", "review"] + }, + "excerpt": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 1024 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?(?:#[^?#\\s]+)?$" + } + ] + }, + "system": { + "type": "boolean" + } + }, + "required": ["id", "excerpt", "url"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "mergeRequest", "comment", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-merge-request-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-merge-request-lifecycle.v1.schema.json new file mode 100644 index 000000000..523482edb --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-merge-request-lifecycle.v1.schema.json @@ -0,0 +1,226 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", + "$ref": "#/$defs/GitLabMergeRequestLifecycleDataV1", + "$defs": { + "GitLabMergeRequestLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "mergeRequest": { + "type": "object", + "properties": { + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "sourceBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "targetBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "commit": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "reviewState": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "action": { + "type": "string", + "enum": ["open", "update", "close", "reopen", "merge", "review"] + } + }, + "required": ["iid", "title", "url", "action"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "mergeRequest", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-pipeline-completed.v1.schema.json b/packages/eventing-core/schemas/gitlab-pipeline-completed.v1.schema.json new file mode 100644 index 000000000..c429067d2 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-pipeline-completed.v1.schema.json @@ -0,0 +1,518 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-pipeline-completed:v1", + "$ref": "#/$defs/GitLabPipelineCompletedDataV1", + "$defs": { + "GitLabPipelineCompletedDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "pipeline": { + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "mergeRequestIid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "detailedStatus": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "ref": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "sha": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "durationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "queuedDurationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "stageCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "status": { + "type": "string", + "enum": ["failed"] + }, + "failedJobCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "failedJobsTruncated": { + "type": "boolean" + }, + "failedJobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "stage": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "status": { + "type": "string", + "enum": ["failed"] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + } + }, + "required": ["id", "name", "status"], + "additionalProperties": false + }, + "allOf": [ + { + "maxItems": 20 + } + ] + } + }, + "required": [ + "id", + "url", + "status", + "failedJobCount", + "failedJobsTruncated", + "failedJobs" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "mergeRequestIid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "detailedStatus": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "ref": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "sha": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "durationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "queuedDurationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "stageCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "status": { + "type": "string", + "enum": ["success", "canceled", "skipped"] + } + }, + "required": ["id", "url", "status"], + "additionalProperties": false + } + ] + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "pipeline", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-project-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-project-lifecycle.v1.schema.json new file mode 100644 index 000000000..548b7097f --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-project-lifecycle.v1.schema.json @@ -0,0 +1,142 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-project-lifecycle:v1", + "$ref": "#/$defs/GitLabProjectLifecycleDataV1", + "$defs": { + "GitLabProjectLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "action": { + "type": "string", + "enum": [ + "created", + "destroyed", + "renamed", + "transferred", + "updated", + "archived", + "unarchived", + "deletion_requested" + ] + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "action", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-ref-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-ref-lifecycle.v1.schema.json new file mode 100644 index 000000000..2587826d7 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-ref-lifecycle.v1.schema.json @@ -0,0 +1,208 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-ref-lifecycle:v1", + "$ref": "#/$defs/GitLabRefLifecycleDataV1", + "$defs": { + "GitLabRefLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "ref": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["branch", "tag"] + }, + "action": { + "type": "string", + "enum": ["create", "update", "delete"] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "before": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "after": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "commitCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "action", "name", "before", "after"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "ref", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-release-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-release-lifecycle.v1.schema.json new file mode 100644 index 000000000..42a85cfeb --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-release-lifecycle.v1.schema.json @@ -0,0 +1,187 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-release-lifecycle:v1", + "$ref": "#/$defs/GitLabReleaseLifecycleDataV1", + "$defs": { + "GitLabReleaseLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "release": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "tag": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "action": { + "type": "string", + "enum": ["create", "update", "delete"] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + } + }, + "required": ["tag", "action"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "release", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/scripts/generate-schemas.ts b/packages/eventing-core/scripts/generate-schemas.ts index 2ea54a516..c507e05a9 100644 --- a/packages/eventing-core/scripts/generate-schemas.ts +++ b/packages/eventing-core/scripts/generate-schemas.ts @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" import { dirname, resolve } from "node:path" import { Schema } from "effect" import { MapleCloudEventSchema, SignalProjectionSpecSchema, SignalScalarSchema } from "../src/model" +import { GITLAB_DATA_SCHEMAS } from "../src/gitlab" const root = resolve(import.meta.dirname, "..") const check = process.argv.includes("--check") @@ -23,6 +24,11 @@ const documents = [ id: "urn:maple:eventing:schema:cloud-event:v1", schema: MapleCloudEventSchema, }, + ...GITLAB_DATA_SCHEMAS.map(([id, schema]) => ({ + path: `schemas/${id.slice("urn:maple:event-schema:".length).replace(":v1", ".v1")}.schema.json`, + id, + schema, + })), ] as const let stale = false diff --git a/packages/eventing-core/src/gitlab.ts b/packages/eventing-core/src/gitlab.ts new file mode 100644 index 000000000..5063bfc18 --- /dev/null +++ b/packages/eventing-core/src/gitlab.ts @@ -0,0 +1,297 @@ +import { Schema } from "effect" + +const Text = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(4 * 1024)) +const CanonicalUrl = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(2 * 1024), + Schema.isPattern(/^https?:\/\/[^/?#@\s]+(?:\/[^?#\s]*)?$/), +) +const CommentUrl = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(2 * 1024), + Schema.isPattern(/^https?:\/\/[^/?#@\s]+(?:\/[^?#\s]*)?(?:#[^?#\s]+)?$/), +) +const PositiveInt64 = Schema.String.check(Schema.isMaxLength(20), Schema.isPattern(/^[1-9][0-9]*$/)) +const NonNegativeInt64 = Schema.String.check(Schema.isMaxLength(20), Schema.isPattern(/^(?:0|[1-9][0-9]*)$/)) +const Revision = Schema.String.check( + Schema.isMinLength(6), + Schema.isMaxLength(64), + Schema.isPattern(/^[0-9a-f]+$/i), +) +const Token = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(64), + Schema.isPattern(/^[a-z][a-z0-9_]*$/), +) +const CommentExcerpt = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(1024)) + +const Project = Schema.Struct({ + id: PositiveInt64, + path: Text, + oldPath: Schema.optionalKey(Text), +}) + +const Actor = Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + name: Schema.optionalKey(Text), + username: Schema.optionalKey(Text), +}) + +const Common = { + sourceEvent: Token, + actor: Schema.optionalKey(Actor), + result: Schema.optionalKey(Text), + serviceName: Schema.optionalKey(Text), +} as const + +const Issue = Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + iid: PositiveInt64, + type: Token, + title: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + state: Schema.optionalKey(Token), + labels: Schema.optionalKey( + Schema.Array(Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256))).check( + Schema.isMaxLength(50), + ), + ), +}) + +const MergeRequest = Schema.Struct({ + iid: PositiveInt64, + title: Text, + url: CanonicalUrl, + sourceBranch: Schema.optionalKey(Text), + targetBranch: Schema.optionalKey(Text), + commit: Schema.optionalKey(Revision), + reviewState: Schema.optionalKey(Token), +}) + +const Comment = Schema.Struct({ + id: PositiveInt64, + kind: Schema.optionalKey(Schema.Literals(["comment", "review"])), + excerpt: CommentExcerpt, + url: CommentUrl, + system: Schema.optionalKey(Schema.Boolean), +}) + +export const GitLabIssueCreatedDataSchema = Schema.Struct({ + project: Schema.Struct({ id: Schema.optionalKey(Schema.String), path: Schema.String }), + issue: Schema.Struct({ + id: Schema.optionalKey(Schema.String), + iid: Schema.String, + title: Schema.optionalKey(Schema.String), + url: Schema.optionalKey(Schema.String), + }), + actor: Schema.optionalKey( + Schema.Struct({ id: Schema.optionalKey(Schema.String), username: Schema.optionalKey(Schema.String) }), + ), + serviceName: Schema.optionalKey(Schema.String), + body: Schema.optionalKey(Schema.Unknown), +}).annotate({ identifier: "GitLabIssueCreatedDataV1" }) + +export const GitLabProjectLifecycleDataSchema = Schema.Struct({ + project: Project, + action: Schema.Literals([ + "created", + "destroyed", + "renamed", + "transferred", + "updated", + "archived", + "unarchived", + "deletion_requested", + ]), + ...Common, +}).annotate({ identifier: "GitLabProjectLifecycleDataV1" }) + +export const GitLabIssueLifecycleDataSchema = Schema.Struct({ + project: Project, + issue: Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + iid: PositiveInt64, + type: Token, + title: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + state: Schema.optionalKey(Token), + labels: Schema.optionalKey( + Schema.Array(Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256))).check( + Schema.isMaxLength(50), + ), + ), + action: Schema.Literals(["open", "update", "close", "reopen"]), + }), + ...Common, +}).annotate({ identifier: "GitLabIssueLifecycleDataV1" }) + +export const GitLabIssueCommentDataSchema = Schema.Struct({ + project: Project, + issue: Issue, + comment: Comment, + ...Common, +}).annotate({ identifier: "GitLabIssueCommentDataV1" }) + +export const GitLabMergeRequestLifecycleDataSchema = Schema.Struct({ + project: Project, + mergeRequest: Schema.Struct({ + iid: PositiveInt64, + title: Text, + url: CanonicalUrl, + sourceBranch: Schema.optionalKey(Text), + targetBranch: Schema.optionalKey(Text), + commit: Schema.optionalKey(Revision), + reviewState: Schema.optionalKey(Token), + action: Schema.Literals(["open", "update", "close", "reopen", "merge", "review"]), + }), + ...Common, +}).annotate({ identifier: "GitLabMergeRequestLifecycleDataV1" }) + +export const GitLabMergeRequestCommentDataSchema = Schema.Struct({ + project: Project, + mergeRequest: MergeRequest, + comment: Comment, + ...Common, +}).annotate({ identifier: "GitLabMergeRequestCommentDataV1" }) + +const PipelineCommon = { + id: PositiveInt64, + iid: Schema.optionalKey(PositiveInt64), + mergeRequestIid: Schema.optionalKey(PositiveInt64), + name: Schema.optionalKey(Text), + source: Schema.optionalKey(Text), + detailedStatus: Schema.optionalKey(Text), + url: CanonicalUrl, + ref: Schema.optionalKey(Text), + sha: Schema.optionalKey(Revision), + durationMs: Schema.optionalKey(NonNegativeInt64), + queuedDurationMs: Schema.optionalKey(NonNegativeInt64), + stageCount: Schema.optionalKey(NonNegativeInt64), +} as const + +const FailedJob = Schema.Struct({ + id: PositiveInt64, + name: Text, + stage: Schema.optionalKey(Text), + status: Schema.Literal("failed"), + url: Schema.optionalKey(CanonicalUrl), +}) + +const Pipeline = Schema.Union([ + Schema.Struct({ + ...PipelineCommon, + status: Schema.Literal("failed"), + failedJobCount: NonNegativeInt64, + failedJobsTruncated: Schema.Boolean, + failedJobs: Schema.Array(FailedJob).check(Schema.isMaxLength(20)), + }), + Schema.Struct({ + ...PipelineCommon, + status: Schema.Literals(["success", "canceled", "skipped"]), + }), +]) + +export const GitLabPipelineCompletedDataSchema = Schema.Struct({ + project: Project, + pipeline: Pipeline, + ...Common, +}).annotate({ identifier: "GitLabPipelineCompletedDataV1" }) + +export const GitLabDeploymentLifecycleDataSchema = Schema.Struct({ + project: Project, + deployment: Schema.Struct({ + id: PositiveInt64, + environment: Text, + status: Schema.Literals(["running", "success", "failed", "canceled", "blocked", "manual"]), + revision: Revision, + url: CanonicalUrl, + }), + ...Common, +}).annotate({ identifier: "GitLabDeploymentLifecycleDataV1" }) + +export const GitLabJobLifecycleDataSchema = Schema.Struct({ + project: Project, + job: Schema.Struct({ + id: PositiveInt64, + name: Text, + status: Schema.Literals([ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled", + ]), + stage: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + pipelineId: Schema.optionalKey(PositiveInt64), + ref: Schema.optionalKey(Text), + revision: Schema.optionalKey(Revision), + durationMs: Schema.optionalKey(NonNegativeInt64), + allowFailure: Schema.optionalKey(Schema.Boolean), + }), + ...Common, +}).annotate({ identifier: "GitLabJobLifecycleDataV1" }) + +export const GitLabRefLifecycleDataSchema = Schema.Struct({ + project: Project, + ref: Schema.Struct({ + type: Schema.Literals(["branch", "tag"]), + action: Schema.Literals(["create", "update", "delete"]), + name: Text, + before: Revision, + after: Revision, + url: Schema.optionalKey(CanonicalUrl), + commitCount: Schema.optionalKey(NonNegativeInt64), + }), + ...Common, +}).annotate({ identifier: "GitLabRefLifecycleDataV1" }) + +export const GitLabReleaseLifecycleDataSchema = Schema.Struct({ + project: Project, + release: Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + tag: Text, + action: Schema.Literals(["create", "update", "delete"]), + name: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + }), + ...Common, +}).annotate({ identifier: "GitLabReleaseLifecycleDataV1" }) + +export type GitLabIssueCreatedData = Schema.Schema.Type +export type GitLabProjectLifecycleData = Schema.Schema.Type +export type GitLabIssueLifecycleData = Schema.Schema.Type +export type GitLabIssueCommentData = Schema.Schema.Type +export type GitLabMergeRequestLifecycleData = Schema.Schema.Type +export type GitLabMergeRequestCommentData = Schema.Schema.Type +export type GitLabPipelineCompletedData = Schema.Schema.Type +export type GitLabDeploymentLifecycleData = Schema.Schema.Type +export type GitLabJobLifecycleData = Schema.Schema.Type +export type GitLabRefLifecycleData = Schema.Schema.Type +export type GitLabReleaseLifecycleData = Schema.Schema.Type + +export const GITLAB_DATA_SCHEMAS = [ + ["urn:maple:event-schema:gitlab-issue-created:v1", GitLabIssueCreatedDataSchema], + ["urn:maple:event-schema:gitlab-project-lifecycle:v1", GitLabProjectLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-issue-lifecycle:v1", GitLabIssueLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-issue-comment:v1", GitLabIssueCommentDataSchema], + ["urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", GitLabMergeRequestLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-merge-request-comment:v1", GitLabMergeRequestCommentDataSchema], + ["urn:maple:event-schema:gitlab-pipeline-completed:v1", GitLabPipelineCompletedDataSchema], + ["urn:maple:event-schema:gitlab-deployment-lifecycle:v1", GitLabDeploymentLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-job-lifecycle:v1", GitLabJobLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-ref-lifecycle:v1", GitLabRefLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-release-lifecycle:v1", GitLabReleaseLifecycleDataSchema], +] as const + +export const validateGitLabEventData = (dataSchema: string, candidate: unknown): unknown => { + const entry = GITLAB_DATA_SCHEMAS.find(([id]) => id === dataSchema) + if (entry === undefined) throw new Error(`unknown GitLab event data schema: ${dataSchema}`) + return Schema.decodeUnknownSync(entry[1] as Schema.Codec)(candidate) +} diff --git a/packages/eventing-core/src/index.ts b/packages/eventing-core/src/index.ts index 87253218d..dffa0fdcd 100644 --- a/packages/eventing-core/src/index.ts +++ b/packages/eventing-core/src/index.ts @@ -1,4 +1,5 @@ export * from "./event" +export * from "./gitlab" export * from "./model" export * from "./predicate" export * from "./registry"