diff --git a/apps/api/package.json b/apps/api/package.json index 574761c04..4b6ea6923 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -34,12 +34,14 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/clickhouse-builder": "workspace:*", "@maple-dev/effect-sdk": "workspace:*", + "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", diff --git a/apps/api/src/planetscale-webhook-runtime.test.ts b/apps/api/src/planetscale-webhook-runtime.test.ts index 6c877a323..08398f091 100644 --- a/apps/api/src/planetscale-webhook-runtime.test.ts +++ b/apps/api/src/planetscale-webhook-runtime.test.ts @@ -1,27 +1,41 @@ import type { MessageBatch } from "@cloudflare/workers-types" import { afterEach, assert, describe, it } from "@effect/vitest" -import { Effect, Layer } from "effect" +import { OrgId } from "@maple/domain/http" +import { Effect, Layer, Schema } from "effect" import { Database, DatabaseError } from "@/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@/platform/test-pglite" import { processPlanetScaleWebhookBatch } from "./planetscale-webhook-runtime" +import { projectPlanetScaleWebhookEvent } from "./services/integrations/planetscale/webhook-events" import type { PlanetScaleWebhookJob } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) -const job: PlanetScaleWebhookJob = { - kind: "planetscale-webhook", - orgId: "org_1", - connectionId: "connection_1", - payload: { +const orgId = Schema.decodeUnknownSync(OrgId)("org_1") + +const makeJob = ( + payload: PlanetScaleWebhookJob["payload"] = { event: "branch.out_of_memory", organization: "acme", database: "shop", resource: { name: "main" }, }, +): PlanetScaleWebhookJob => ({ + kind: "planetscale-webhook", + orgId, + connectionId: "connection_1", + payload, receivedAt: 1_000, -} + event: projectPlanetScaleWebhookEvent({ + orgId, + connectionId: "connection_1", + payload, + receivedAt: 1_000, + }), +}) + +const job = makeJob() const makeBatch = (body: unknown) => { let acknowledged = false @@ -70,6 +84,32 @@ describe("PlanetScale webhook queue consumer", () => { }).pipe(Effect.provide(testDb.layer)) }) + it.effect("processes the exact pre-event-envelope queue body during rolling upgrades", () => { + const testDb = createTestDb(trackedDbs) + const legacyJob = { + kind: "planetscale-webhook", + orgId, + connectionId: "connection_1", + payload: job.payload, + receivedAt: 1_000, + } + const delivery = makeBatch(legacyJob) + return Effect.gen(function* () { + yield* processPlanetScaleWebhookBatch(delivery.batch) + assert.isTrue(delivery.acknowledged()) + assert.isFalse(delivery.retried()) + const row = yield* Effect.promise(() => + queryFirstRow<{ workflow_state: string; occurrence_count: number }>( + testDb, + "SELECT workflow_state, occurrence_count FROM error_issues WHERE org_id = $1", + ["org_1"], + ), + ) + assert.strictEqual(row?.workflow_state, "triage") + assert.strictEqual(row?.occurrence_count, 1) + }).pipe(Effect.provide(testDb.layer)) + }) + it.effect("acknowledges terminal malformed jobs", () => { const testDb = createTestDb(trackedDbs) const delivery = makeBatch({ kind: "not-a-planetscale-job" }) @@ -86,10 +126,7 @@ describe("PlanetScale webhook queue consumer", () => { it.effect("writes a lifecycle event to the timeline but not to the issue hub", () => { const testDb = createTestDb(trackedDbs) - const delivery = makeBatch({ - ...job, - payload: { ...job.payload, event: "branch.ready" }, - }) + const delivery = makeBatch(makeJob({ ...job.payload, event: "branch.ready" })) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) assert.isTrue(delivery.acknowledged()) @@ -138,14 +175,13 @@ describe("PlanetScale webhook queue consumer", () => { it.effect("carries the deploy-request number so redelivery dedupes", () => { const testDb = createTestDb(trackedDbs) - const delivery = makeBatch({ - ...job, - payload: { + const delivery = makeBatch( + makeJob({ ...job.payload, event: "deploy_request.schema_applied", resource: { number: 42 }, - }, - }) + }), + ) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) const event = yield* Effect.promise(() => diff --git a/apps/api/src/planetscale-webhook-runtime.ts b/apps/api/src/planetscale-webhook-runtime.ts index 71bc6c596..544f702c5 100644 --- a/apps/api/src/planetscale-webhook-runtime.ts +++ b/apps/api/src/planetscale-webhook-runtime.ts @@ -10,9 +10,10 @@ import { deployRequestNumber, insertPlanetScaleEvent, planetScaleBranchName, + projectPlanetScaleWebhookEvent, upsertPlanetScaleIssue, } from "./services/integrations/planetscale/webhook-events" -import { PlanetScaleWebhookJob } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" +import { PlanetScaleWebhookQueueMessage } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" const telemetry = MapleCloudflareSDK.make({ serviceName: "maple-api", @@ -32,7 +33,7 @@ export const buildPlanetScaleWebhookLayer = (_env: Record) => { export const flushPlanetScaleWebhookTelemetry = (env: Record) => telemetry.flush(env) -const decodeJob = Schema.decodeUnknownEffect(PlanetScaleWebhookJob) +const decodeJob = Schema.decodeUnknownEffect(PlanetScaleWebhookQueueMessage) export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => Effect.forEach( @@ -54,9 +55,29 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => ), ), onSuccess: (job) => { - const classified = classifyPlanetScaleEvent(job.payload.event) + // Old jobs can remain in Cloudflare Queue across a deploy. Rebuild the + // event from the durable legacy fields instead of malformed-acking them. + const event = + job.event ?? + projectPlanetScaleWebhookEvent({ + orgId: job.orgId, + connectionId: job.connectionId, + payload: job.payload, + receivedAt: job.receivedAt, + }) + const eventData = + typeof event.data === "object" && + event.data !== null && + !Array.isArray(event.data) + ? (event.data as { readonly [key: string]: unknown }) + : null + const eventName = + typeof eventData?.event === "string" ? eventData.event : job.payload.event + const classified = classifyPlanetScaleEvent(eventName) const annotateJob = Effect.annotateCurrentSpan({ orgId: job.orgId, + "maple.event.id": event.id, + "maple.event.type": event.type, "maple.planetscale.connection_id": job.connectionId, "maple.planetscale.webhook.event": job.payload.event, }) diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts index 3b22e91f4..3d5bd92ba 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts @@ -187,11 +187,13 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(jobs[0]?.orgId, "org_1") assert.strictEqual(jobs[0]?.connectionId, CONNECTION_ID) assert.strictEqual(jobs[0]?.payload.event, "branch.out_of_memory") + assert.strictEqual(jobs[0]?.event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(jobs[0]?.event.tenantid, "org_1") }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) - it.effect("enqueues lifecycle events too, and still drops genuinely unknown ones", () => { + it.effect("enqueues every verified factual event before downstream classification", () => { const testDb = createTestDb(trackedDbs) const jobs: PlanetScaleWebhookJob[] = [] return Effect.gen(function* () { @@ -260,15 +262,16 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(branchReady.status, 202) assert.strictEqual(jobs.length, 2) - // Forward-compatibility must not become "enqueue everything": an - // event neither side knows is acknowledged and dropped. + // Unknown provider facts also enter the typed event layer. The current + // issue/timeline consumer may ignore them, but other consumers can opt in. const unknown = yield* post({ event: "branch.some_future_event", organization: "acme", database: "shop", }) assert.strictEqual(unknown.status, 202) - assert.strictEqual(jobs.length, 2) + assert.strictEqual(jobs.length, 3) + assert.strictEqual(jobs[2]?.payload.event, "branch.some_future_event") }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.ts b/apps/api/src/routes/v1/planetscale-webhook.http.ts index f09355f6b..c5d4c3b29 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.ts @@ -9,6 +9,7 @@ import { Env } from "@/platform/Env" import { classifyPlanetScaleEvent, decodePlanetScaleWebhookPayload, + projectPlanetScaleWebhookEvent, verifyPlanetScaleSignature, } from "@/services/integrations/planetscale/webhook-events" import { PlanetScaleWebhookQueue } from "@/services/integrations/planetscale/PlanetScaleWebhookQueue" @@ -163,17 +164,33 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => return textResponse("ok", 200) } - // Both issue-worthy and timeline-only events go through the queue: the - // durable retry is what makes a missed deploy marker recoverable. - if (classified.action === "issue" || classified.action === "timeline") { + // Every verified factual event is normalized and projected before the + // durable queue boundary. The queued CloudEvent is the stable contract; + // payload remains temporarily for parity with the existing consumers. + { const now = yield* Clock.currentTimeMillis + const orgId = decodeOrgIdSync(connection.orgId) + const event = yield* Effect.try({ + try: () => + projectPlanetScaleWebhookEvent({ + orgId, + connectionId, + payload, + receivedAt: now, + }), + catch: () => + new PlanetScaleWebhookUnavailable({ + message: "Webhook event projection unavailable", + }), + }) const enqueued = yield* webhookQueue .send({ kind: "planetscale-webhook", - orgId: decodeOrgIdSync(connection.orgId), + orgId, connectionId, payload, receivedAt: now, + event, }) .pipe( Effect.tapError((error) => @@ -198,10 +215,6 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => event: payload.event, }), ) - } else { - yield* Effect.logInfo("PlanetScale webhook lifecycle event acknowledged").pipe( - Effect.annotateLogs({ orgId: connection.orgId, event: payload.event }), - ) } yield* Effect.annotateCurrentSpan({ diff --git a/apps/api/src/services/alerts/AlertDestinationDelivery.ts b/apps/api/src/services/alerts/AlertDestinationDelivery.ts index 0e0e2eb50..212ba740b 100644 --- a/apps/api/src/services/alerts/AlertDestinationDelivery.ts +++ b/apps/api/src/services/alerts/AlertDestinationDelivery.ts @@ -7,6 +7,7 @@ import { type AlertIncidentId, type AlertRuleId, } from "@maple/domain/http" +import { projectAlertLifecycleEvent } from "@maple/alerting-core" import type { AlertDestinationRow } from "@maple/db" import { Effect } from "effect" import { parseBase64Aes256GcmKey } from "@/platform/Crypto" @@ -131,8 +132,26 @@ export const makeAlertDestinationDelivery = (options: { { sendEmail, resolveSlackBotToken: options.resolveSlackBotToken }, ) - const buildPayload = (context: AlertDeliveryPayloadContext) => + const buildPayload = (context: AlertDeliveryPayloadContext, tenantId: string) => ({ + event: projectAlertLifecycleEvent({ + tenantId, + ruleId: context.ruleId, + ruleName: context.ruleName, + incidentId: context.incidentId, + eventType: context.eventType, + incidentStatus: context.incidentStatus, + groupKey: context.groupKey, + signalType: context.signalType, + severity: context.severity, + comparator: context.comparator, + threshold: context.threshold, + thresholdUpper: context.thresholdUpper, + windowMinutes: context.windowMinutes, + value: context.value, + sampleCount: context.sampleCount, + occurredAtMs: context.sentAtMs, + }), eventType: context.eventType, incidentId: context.incidentId, incidentStatus: context.incidentStatus, @@ -157,6 +176,7 @@ export const makeAlertDestinationDelivery = (options: { chatUrl: buildAlertChatUrl(options.appBaseUrl, context), sentAt: new Date(context.sentAtMs).toISOString(), }) satisfies { + readonly event: ReturnType readonly eventType: AlertDeliveryPayloadContext["eventType"] readonly incidentId: AlertIncidentId | null readonly incidentStatus: AlertDeliveryPayloadContext["incidentStatus"] @@ -181,7 +201,7 @@ export const makeAlertDestinationDelivery = (options: { secretConfig: enrichedSecret, ...context, } - const payload = buildPayload(fullContext) + const payload = buildPayload(fullContext, destinationRow.orgId) return yield* dispatchDelivery(fullContext, JSON.stringify(payload)) }) diff --git a/apps/api/src/services/alerts/AlertsService.test.ts b/apps/api/src/services/alerts/AlertsService.test.ts index bd30f5b39..462a770d0 100644 --- a/apps/api/src/services/alerts/AlertsService.test.ts +++ b/apps/api/src/services/alerts/AlertsService.test.ts @@ -1,6 +1,7 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { Cause, Clock, ConfigProvider, Duration, Effect, Exit, Layer, Option, Schema } from "effect" import { TestClock } from "effect/testing" +import { projectAlertLifecycleEvent } from "@maple/alerting-core" import { AlertDestinationInUseError, AlertForbiddenError, @@ -1573,6 +1574,24 @@ describe("AlertsService", () => { const userId = asUserId("user_timeout") const destination = yield* createWebhookDestination(alerts, orgId, userId) const rule = yield* createErrorRateRule(alerts, orgId, userId, destination.id) + const lifecycleEvent = projectAlertLifecycleEvent({ + tenantId: orgId, + ruleId: rule.id, + ruleName: rule.name, + incidentId: null, + eventType: "test", + incidentStatus: "resolved", + groupKey: null, + signalType: rule.signalType, + severity: rule.severity, + comparator: rule.comparator, + threshold: rule.threshold, + thresholdUpper: rule.thresholdUpper, + windowMinutes: rule.windowMinutes, + value: 0, + sampleCount: 0, + occurredAtMs: fixedTime, + }) yield* Effect.promise(() => insertDeliveryEventRow(testDb, { @@ -1587,6 +1606,7 @@ describe("AlertsService", () => { status: "queued", scheduledAt: fixedTime - 1, payloadJson: JSON.stringify({ + event: lifecycleEvent, eventType: "test", incidentId: null, incidentStatus: "resolved", @@ -1607,6 +1627,7 @@ describe("AlertsService", () => { }, linkUrl: "http://127.0.0.1:3471/alerts", sentAt: new Date(fixedTime).toISOString(), + futureAdditiveField: { preserve: true }, }), }), ) @@ -1615,6 +1636,18 @@ describe("AlertsService", () => { // live runtime clock, so the timeout fires on its own in real time. const tick = yield* alerts.runSchedulerTick() const events = yield* alerts.listDeliveryEvents(orgId) + const retryPayload = yield* Effect.promise(() => + queryFirstRow<{ + payload_json: { + event?: unknown + futureAdditiveField?: unknown + } + }>( + testDb, + "select payload_json from alert_delivery_events where delivery_key = $1 and attempt_number = 2", + ["timeout-delivery-key"], + ), + ) assert.strictEqual(tick.processedCount, 1) assert.strictEqual(tick.deliveryFailureCount, 1) @@ -1627,6 +1660,8 @@ describe("AlertsService", () => { assert.strictEqual(timeoutEvent?.status, "failed") assert.include(timeoutEvent?.errorMessage ?? "", "timed out") assert.strictEqual(retryEvent?.status, "queued") + assert.deepStrictEqual(retryPayload?.payload_json.event, lifecycleEvent) + assert.deepStrictEqual(retryPayload?.payload_json.futureAdditiveField, { preserve: true }) }).pipe( Effect.provide( makeLayer(testDb, makeWarehouseStub({ tracesAggregateRows: emptyWarehouseRows }), { diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index f88c09434..b950fb054 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -1,6 +1,17 @@ +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + type AlertLifecycleInput, +} from "@maple/alerting-core" import { formatWarehouseDateTime } from "@maple/query-engine" +import { MapleCloudEventSchema } from "@maple/eventing-core" import { AlertComparator as AlertComparatorSchema, + type AlertComparator, AlertDeliveryError, AlertDestinationDecryptionError, AlertDeliveryEventDocument, @@ -26,7 +37,6 @@ import { AlertSignalType as AlertSignalTypeSchema, AlertValidationError, AlertNotificationTemplate, - type AlertComparator, type AlertDestinationType, type AlertEventType as AlertEventTypeValue, type AlertRuleUpsertRequest, @@ -137,7 +147,6 @@ interface DeliveryAttemptFailure { readonly retryable: boolean } -const MAX_DELIVERY_ATTEMPTS = 5 const ALERT_TEST_DELIVERY_CONCURRENCY = 5 const ALERT_CHECK_INGEST_CONCURRENCY = 4 // Storm fuse: cap issue-hub upserts per scheduler tick so a pathological @@ -153,6 +162,7 @@ type DatabaseExecutor = DatabaseClient | DatabaseTransaction /* -------------------------------------------------------------------------- */ const StoredDeliveryPayloadSchema = Schema.Struct({ + event: Schema.optionalKey(MapleCloudEventSchema), eventType: Schema.optionalKey(Schema.String), incidentId: Schema.optionalKey(Schema.NullOr(Schema.String)), incidentStatus: Schema.optionalKey(Schema.String), @@ -212,25 +222,7 @@ const MAX_PREVIEW_BUCKETS = 200 /** Preserve each org's oldest-first order while preventing one org from monopolizing a tick. */ export const interleaveAlertRulesByOrg = ( rows: ReadonlyArray, -): ReadonlyArray => { - const queues = new Map() - for (const row of rows) { - const queue = queues.get(row.orgId) - if (queue) queue.push(row) - else queues.set(row.orgId, [row]) - } - - const fair: T[] = [] - let index = 0 - while (fair.length < rows.length) { - for (const queue of queues.values()) { - const row = queue[index] - if (row !== undefined) fair.push(row) - } - index += 1 - } - return fair -} +): ReadonlyArray => interleaveAlertRulesByTenant(rows, (row) => row.orgId) // Tinybird DateTime64(3) wire format for alert_checks ingest: // "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). @@ -240,26 +232,6 @@ const toIngestDateTime64 = (epochMs: number) => { return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` } -const compareThreshold = ( - value: number, - comparator: AlertComparator, - threshold: number, - thresholdUpper: number | null = null, -): boolean => - Match.value(comparator).pipe( - Match.when("gt", () => value > threshold), - Match.when("gte", () => value >= threshold), - Match.when("lt", () => value < threshold), - Match.when("lte", () => value <= threshold), - Match.when("eq", () => value === threshold), - Match.when("neq", () => value !== threshold), - Match.when("between", () => thresholdUpper != null && value >= threshold && value <= thresholdUpper), - Match.when( - "not_between", - () => thresholdUpper != null && (value < threshold || value > thresholdUpper), - ), - Match.exhaustive, - ) const makeDeliveryError = (message: string, destinationType?: AlertDestinationType, cause?: unknown) => new AlertDeliveryError({ @@ -495,79 +467,26 @@ export class AlertsService extends Context.Service, reasonOverride?: string, - ): EvaluatedRule => { - const noDataBehavior = rule.compiledPlan.noDataBehavior - // Sample-weighted counts arrive fractional from the warehouse - // (`sum(SampleRate)`), and this flows into `last_sample_count`, an - // `integer` column — an unrounded value fails the insert outright. - const sampleCount = Math.round(obs.sampleCount) - const value = obs.hasData ? obs.value : noDataBehavior === "zero" ? 0 : null - - if (!obs.hasData && noDataBehavior === "skip") { - return { - status: "skipped", - value: null, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: "No data in the selected window", - // Inert: `skipped` never resolves an incident, so this branch - // short-circuits before any status is derived from a synthesized value. - derivedFromNoData: false, - } - } - - if (sampleCount < rule.minimumSampleCount) { - return { - status: "skipped", - value, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, + ): EvaluatedRule => + evaluateAlertObservation( + { comparator: rule.comparator, - reason: `Sample count ${sampleCount} is below minimum ${rule.minimumSampleCount}`, - derivedFromNoData: false, - } - } - - if (value == null) { - return { - status: "skipped", - value: null, - sampleCount, threshold: rule.threshold, thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: "Alert evaluation did not return a scalar value", - derivedFromNoData: false, - } - } - - return { - status: compareThreshold(value, rule.comparator, rule.threshold, rule.thresholdUpper) - ? "breached" - : "healthy", - value, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: - reasonOverride ?? + minimumSampleCount: rule.minimumSampleCount, + noDataBehavior: rule.compiledPlan.noDataBehavior, + }, + obs, + reasonOverride ?? `${rule.signalType} ${formatComparator(rule.comparator, rule.threshold, rule.thresholdUpper)}`, - // Only reachable with `noDataBehavior: "zero"` — the "skip" branch - // returned above. The comparison ran against a fabricated 0. - derivedFromNoData: !obs.hasData, - } - } + ) const buildDeliveryKey = ( incidentId: string, destinationId: string, eventType: AlertEventTypeValue, scheduledAt: number, - ) => [incidentId, destinationId, eventType, scheduledAt].join(":") + ) => makeAlertDeliveryKey(incidentId, destinationId, eventType, scheduledAt) const insertDeliveryEventRecord = ( db: DatabaseExecutor, @@ -685,26 +604,29 @@ export class AlertsService extends Context.Service [row.id, row])) - const payload = buildPayload({ - eventType, - incidentId: incident.id, - incidentStatus: decodeAlertIncidentStatusSync(incident.status), - dedupeKey: incident.dedupeKey, - ruleId: rule.id, - ruleName: rule.name, - groupKey: incident.groupKey, - signalType: rule.signalType, - severity: rule.severity, - comparator: rule.comparator, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - windowMinutes: rule.windowMinutes, - value: evaluation.value, - sampleCount: evaluation.sampleCount, - template: rule.notificationTemplate, - linkUrl: resolveNotificationLinkUrl(rule, incident.groupKey), - sentAtMs: scheduledAt, - }) + const payload = buildPayload( + { + eventType, + incidentId: incident.id, + incidentStatus: decodeAlertIncidentStatusSync(incident.status), + dedupeKey: incident.dedupeKey, + ruleId: rule.id, + ruleName: rule.name, + groupKey: incident.groupKey, + signalType: rule.signalType, + severity: rule.severity, + comparator: rule.comparator, + threshold: rule.threshold, + thresholdUpper: rule.thresholdUpper, + windowMinutes: rule.windowMinutes, + value: evaluation.value, + sampleCount: evaluation.sampleCount, + template: rule.notificationTemplate, + linkUrl: resolveNotificationLinkUrl(rule, incident.groupKey), + sentAtMs: scheduledAt, + }, + orgId, + ) yield* Effect.forEach( rule.destinationIds, @@ -736,9 +658,8 @@ export class AlertsService extends Context.Service row.payloadJson as Record), Effect.orElseSucceed(() => ({})), - )) as Record + ) yield* insertDeliveryEvent( row.orgId, row.incidentId, @@ -1465,7 +1390,7 @@ export class AlertsService extends Context.Service= against *Required, so saturating keeps open/resolve behavior - // identical while letting steady-state ticks skip the state upsert above. - const consecutiveBreaches = - evaluation.status === "breached" - ? Math.min( - (state?.consecutiveBreaches ?? 0) + 1, - normalized.consecutiveBreachesRequired, - ) - : 0 - const consecutiveHealthy = - evaluation.status === "healthy" - ? Math.min( - (state?.consecutiveHealthy ?? 0) + 1, - normalized.consecutiveHealthyRequired, - ) - : 0 - + let lifecycle = planAlertLifecycle(lifecycleInput) + // Persist the counter/state decision before follow-up adapter work, as + // before extraction. A failed flap-history or liveness query must not + // discard an evaluation that already completed successfully. yield* upsertState({ - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, lastStatus: evaluation.status, lastValue: evaluation.value, lastSampleCount: evaluation.sampleCount, }) - if ( - evaluation.status === "breached" && - openIncident == null && - consecutiveBreaches >= normalized.consecutiveBreachesRequired - ) { - // Flap suppression: a metric oscillating around the threshold opens - // a fresh incident per flap, which would email an identical trigger - // notification every few minutes. If the previous incident for this - // (rule, group) was notified within the renotify interval, open the - // incident but skip the trigger notification and carry the prior - // lastNotifiedAt forward — the renotify gate then enforces one - // email per interval while the flapping persists. + // Ask the persistence adapter for flap history only when the pure core + // has decided that a new incident is otherwise ready to open. + if (lifecycle.transition === "opened") { const priorNotified = (yield* dbExecute((db) => db @@ -1644,9 +1557,46 @@ export class AlertsService extends Context.Service db.insert(alertIncidents).values(incident)) - if (flapSuppressedAt != null) { + if (lifecycle.notificationSuppression === "flapping") { yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe( Effect.annotateLogs({ ruleId: row.id, incidentId, groupKey, - priorNotifiedAt: flapSuppressedAt.toISOString(), + priorNotifiedAt: inheritedNotificationAt?.toISOString(), }), ) - } else { + } else if (lifecycle.eventType === "trigger") { yield* queueIncidentNotifications( row.orgId, normalized, incident, evaluation, - "trigger", + lifecycle.eventType, timestamp, ) } return { - transition: "opened" as const, + transition: lifecycle.transition, incidentId, openedIncidentId: incidentId, - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, } } - if (evaluation.status === "breached" && openIncident != null) { + if (lifecycle.transition === "continued" && openIncident != null) { const refreshedIncident = { ...openIncident, lastTriggeredAt: new Date(timestamp), @@ -1715,19 +1662,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -1737,72 +1671,32 @@ export class AlertsService extends Context.Service= normalized.consecutiveHealthyRequired - ) { - // A "healthy" synthesized from an empty window is a statement - // about missing data, not about a recovered system: with - // `noDataBehavior: "zero"` a total ingest outage compares as 0 < - // threshold and would resolve every incident it touches, paging - // out a wave of false all-clears. Believe it only once telemetry - // is provably still arriving. - if (evaluation.derivedFromNoData) { - const liveness = yield* telemetryStillFlowing( - row.orgId, - normalized, - openIncident.firstTriggeredAt.getTime(), - timestamp, - ) - if (!liveness.dataFlowing) { - yield* Effect.logWarning( - "Holding incident open: healthy evaluation came from missing telemetry", - ).pipe( - Effect.annotateLogs({ - orgId: row.orgId, - ruleId: row.id, - incidentId: openIncident.id, - groupKey, - livenessReason: liveness.reason, - observedCount: liveness.observedCount, - baselineCount: liveness.baselineCount, - }), - ) - return { - transition: "none" as const, - incidentId: carriedIncidentId, - openedIncidentId: null, - consecutiveBreaches, - consecutiveHealthy, - } - } - } - + if (lifecycle.transition === "resolved" && openIncident != null) { const resolvedIncident = { ...openIncident, status: "resolved" as const, @@ -1812,7 +1706,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -1826,14 +1719,7 @@ export class AlertsService extends Context.Service) => diff --git a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts index e4b5bd433..6685d71da 100644 --- a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts +++ b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts @@ -1,20 +1,36 @@ import type { Queue } from "@cloudflare/workers-types" import { OrgId } from "@maple/domain/http" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { MapleCloudEventSchema } from "@maple/eventing-core" import { Context, Effect, Layer, Schema } from "effect" import { PlanetScaleWebhookPayload } from "./webhook-events" const QUEUE_BINDING = "PLANETSCALE_WEBHOOK_QUEUE" -export const PlanetScaleWebhookJob = Schema.Struct({ +const PlanetScaleWebhookJobFields = { kind: Schema.Literal("planetscale-webhook"), orgId: OrgId, connectionId: Schema.String, payload: PlanetScaleWebhookPayload, receivedAt: Schema.Number, +} as const + +/** Exact queue body emitted before the typed CloudEvent migration. */ +export const LegacyPlanetScaleWebhookJob = Schema.Struct(PlanetScaleWebhookJobFields) + +/** Current producer contract. New writers must always include the event. */ +export const PlanetScaleWebhookJob = Schema.Struct({ + ...PlanetScaleWebhookJobFields, + event: MapleCloudEventSchema, }) export type PlanetScaleWebhookJob = Schema.Schema.Type +/** Consumer contract kept backward-compatible during rolling deployments. */ +export const PlanetScaleWebhookQueueMessage = Schema.Struct({ + ...PlanetScaleWebhookJobFields, + event: Schema.optionalKey(MapleCloudEventSchema), +}) + export class PlanetScaleWebhookQueueError extends Schema.TaggedError()( "@maple/api/services/planetscale/PlanetScaleWebhookQueueError", { diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts index eb4a0f1d7..1a1383a1d 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts @@ -10,6 +10,7 @@ import { deployRequestNumber, insertPlanetScaleEvent, planetScaleIssueFingerprint, + projectPlanetScaleWebhookEvent, truncateToSecond, upsertPlanetScaleIssue, verifyPlanetScaleSignature, @@ -47,6 +48,63 @@ describe("verifyPlanetScaleSignature", () => { }) describe("classifyPlanetScaleEvent", () => { + it("normalizes queued webhooks into deterministic common CloudEvents", () => { + const payload = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const input = { + orgId: "org_events", + connectionId: "connection-1", + payload, + receivedAt: 1_698_252_880_000, + } + const event = projectPlanetScaleWebhookEvent(input) + assert.deepStrictEqual(event, projectPlanetScaleWebhookEvent(input)) + assert.strictEqual(event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(event.tenantid, "org_events") + assert.strictEqual(event.subject, "planetscale-databases/main-db") + assert.strictEqual((event.data as { readonly event: string }).event, "branch.out_of_memory") + assert.throws( + () => projectPlanetScaleWebhookEvent({ ...input, receivedAt: Number.MAX_SAFE_INTEGER }), + /outside the supported date range/, + ) + }) + + it("keeps source-timestamp identities stable and receipt-time fallbacks payload-consistent", () => { + const timestamped = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const first = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_880_000, + }) + const redelivery = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_990_000, + }) + assert.strictEqual(first.id, redelivery.id) + assert.strictEqual(first.time, redelivery.time) + + const withoutTimestamp = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)({ + event: "branch.ready", + database: "main-db", + }) + const receivedFirst = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: withoutTimestamp, + receivedAt: 1_698_252_880_000, + }) + const receivedAgain = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: withoutTimestamp, + receivedAt: 1_698_252_990_000, + }) + assert.notStrictEqual(receivedFirst.id, receivedAgain.id) + assert.notStrictEqual(receivedFirst.time, receivedAgain.time) + }) + it("maps health events to issues and lifecycle events to timeline rows", () => { assert.strictEqual(classifyPlanetScaleEvent("branch.out_of_memory").action, "issue") assert.strictEqual(classifyPlanetScaleEvent("branch.anomaly").action, "issue") diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.ts b/apps/api/src/services/integrations/planetscale/webhook-events.ts index d63e173d6..dc8d0af63 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.ts @@ -1,4 +1,15 @@ -import { createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { + canonicalJson, + CompiledProjectionRegistry, + defineSignalFields, + ProjectorRegistry, + SignalSourceRegistry, + type JsonValue, + type MapleCloudEvent, + type SignalProjector, + type SignalSourceAdapter, +} from "@maple/eventing-core" import type { IssueSeverity, OrgId, WorkflowState } from "@maple/domain/http" import { ActorId, ErrorIssueEventId, ErrorIssueId } from "@maple/domain/primitives" import { @@ -49,6 +60,152 @@ export const decodePlanetScaleWebhookPayload = Schema.decodeUnknownEffect( Schema.fromJsonString(PlanetScaleWebhookPayload), ) +export interface PlanetScaleWebhookEventInput { + readonly orgId: string + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload + readonly receivedAt: number +} + +interface PlanetScaleWebhookAdapterInput { + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload +} + +interface PlanetScaleWebhookAdapterContext { + readonly tenantId: string + readonly acceptedAt: string +} + +const validDate = (epochMs: number, label: string): Date => { + if (!Number.isSafeInteger(epochMs) || epochMs < 0) + throw new Error(`${label} must be a non-negative epoch millisecond`) + const date = new Date(epochMs) + if (Number.isNaN(date.getTime())) throw new Error(`${label} is outside the supported date range`) + return date +} + +export const PLANETSCALE_WEBHOOK_ADAPTER: SignalSourceAdapter< + PlanetScaleWebhookAdapterInput, + PlanetScaleWebhookAdapterContext +> = { + definition: { + sourceKind: "planetscale.webhook", + fields: [ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "unavailable", + }, + ], + }, + normalize: ({ connectionId, payload }, context) => { + const observedAtDate = new Date(context.acceptedAt) + if (Number.isNaN(observedAtDate.getTime())) + throw new Error("PlanetScale receipt time is outside the supported date range") + const payloadJson = payload as unknown as JsonValue + const occurredAtMs = + payload.timestamp != null && payload.timestamp > 0 + ? Math.trunc(payload.timestamp * 1_000) + : observedAtDate.getTime() + const occurredAt = validDate(occurredAtMs, "PlanetScale event timestamp").toISOString() + const occurrenceId = `derived:sha256:${createHash("sha256") + .update(connectionId) + .update("\0") + .update(canonicalJson(payloadJson)) + .update("\0") + .update(occurredAt) + .digest("hex")}` + return [ + { + sourceKind: "planetscale.webhook", + source: `urn:maple:planetscale:${connectionId}`, + tenantId: context.tenantId, + occurrenceId, + identityQuality: "derived", + occurredAt, + observedAt: observedAtDate.toISOString(), + subject: + payload.database == null + ? `planetscale-connections/${connectionId}` + : `planetscale-databases/${payload.database}`, + fields: defineSignalFields([ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: payload.event }, + }, + ]), + data: { + connectionId, + event: payload.event, + organization: payload.organization ?? null, + database: payload.database ?? null, + resource: (payload.resource ?? null) as JsonValue, + }, + }, + ] + }, +} + +const PLANETSCALE_WEBHOOK_PROJECTOR: SignalProjector> = { + id: "planetscale.webhook", + version: 1, + sourceKinds: ["planetscale.webhook"], + outputType: "dev.maple.planetscale.webhook.received.v1", + dataSchema: "urn:maple:event-schema:planetscale-webhook:v1", + decodeConfig: (value) => { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + Object.keys(value).length > 0 + ) + throw new Error("PlanetScale webhook projector config must be empty") + return {} + }, + project: (signal) => ({ data: signal.data as JsonValue }), +} + +const PLANETSCALE_SOURCES = new SignalSourceRegistry().register(PLANETSCALE_WEBHOOK_ADAPTER.definition) +const PLANETSCALE_PROJECTORS = new ProjectorRegistry().register(PLANETSCALE_WEBHOOK_PROJECTOR) + +/** Normalize and project one verified, durably queued PlanetScale webhook through the common layer. */ +export const projectPlanetScaleWebhookEvent = (input: PlanetScaleWebhookEventInput): MapleCloudEvent => { + const observedAt = validDate(input.receivedAt, "PlanetScale receipt time").toISOString() + const [signal] = PLANETSCALE_WEBHOOK_ADAPTER.normalize( + { connectionId: input.connectionId, payload: input.payload }, + { tenantId: input.orgId, acceptedAt: observedAt }, + ) + if (!signal) throw new Error("PlanetScale webhook adapter produced no signal") + const registry = CompiledProjectionRegistry.compile( + [ + { + id: "planetscale-webhook", + revision: 1, + enabled: true, + tenantId: input.orgId, + sourceKind: "planetscale.webhook", + selector: { + op: "exists", + field: { namespace: "signal", key: "event.name", type: "string" }, + }, + projector: { id: "planetscale.webhook", version: 1, config: {} }, + activeFrom: observedAt, + }, + ], + PLANETSCALE_SOURCES, + PLANETSCALE_PROJECTORS, + ) + const result = registry.evaluate(signal) + if (result.failures.length > 0) throw new Error(result.failures[0]!.message) + if (result.events.length !== 1) throw new Error("PlanetScale webhook projection produced no event") + return result.events[0]! +} + +// --------------------------------------------------------------------------- +// Classification +// --------------------------------------------------------------------------- /** Where an event belongs on the timeline. Mirrored by the web vocabulary table. */ export type PlanetScaleEventCategory = "deploy_request" | "branch" | "database" | "cluster" | "keyspace" diff --git a/apps/cli/package.json b/apps/cli/package.json index 59409a75f..650cdf5ff 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -14,6 +14,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1" diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index d5ed36386..6214ad89f 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto" +import { createHash, randomUUID } from "node:crypto" import { spawnSync } from "node:child_process" import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { cp, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises" @@ -18,6 +18,11 @@ import { syncDirectory, syncTree, } from "./durable-files" +import { + eventingControlSnapshotPath, + LocalEventingControlStore, + type EventingControlSnapshotValidation, +} from "./eventing/control-store" import { CURRENT_LOCAL_SCHEMA, SCHEMA_FINGERPRINT } from "./schema-identity" import schemaSql from "./schema/local-schema.sql" with { type: "text" } import { @@ -28,12 +33,12 @@ import { } from "./store-version" const STATE_FORMAT_VERSION = 1 -const MANIFEST_FORMAT_VERSION = 1 +const MANIFEST_FORMAT_VERSION = 2 const OPERATION_FORMAT_VERSION = 1 const RESTORE_TRANSACTION_FORMAT_VERSION = 1 const RESET_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i -const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "status", "store", "tmp"]) +const RESETTABLE_LIVE_ENTRIES = new Set(["control", "data", "metadata", "status", "store", "tmp"]) export const CHECKPOINT_REOPEN_PROBE_ENV = "MAPLE_INTERNAL_CHECKPOINT_REOPEN_DATA_DIR" const CheckpointUuid = Schema.String.check(Schema.isPattern(CHECKPOINT_ID)) @@ -83,8 +88,7 @@ const CheckpointValidationSchema = Schema.Struct({ export type CheckpointValidation = Schema.Schema.Type -const CheckpointManifestSchema = Schema.Struct({ - formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), +const CheckpointManifestFields = { checkpointId: CheckpointId, operationId: CheckpointOperationId, mapleVersion: Schema.String, @@ -95,8 +99,28 @@ const CheckpointManifestSchema = Schema.Struct({ backupRelativePath: Schema.String, backupBytes: NonNegativeInt, validation: CheckpointValidationSchema, +} as const + +const EventingControlSnapshotValidationSchema = Schema.Struct({ + schemaVersion: NonNegativeInt, + projectionRevisions: NonNegativeInt, + projectionFailures: NonNegativeInt, + stagedEvents: NonNegativeInt, + readyEvents: NonNegativeInt, }) +const CheckpointManifestSchema = Schema.Union([ + Schema.Struct({ formatVersion: Schema.Literal(1), ...CheckpointManifestFields }), + Schema.Struct({ + formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), + ...CheckpointManifestFields, + controlRelativePath: Schema.String, + controlBytes: NonNegativeInt, + controlSha256: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + controlValidation: EventingControlSnapshotValidationSchema, + }), +]) + export type CheckpointManifest = Schema.Schema.Type const CheckpointStateSchema = Schema.Struct({ @@ -124,7 +148,7 @@ const RestoreTransactionPhase = Schema.Literals([ "markers-committed", ]) const ResetTransactionPhase = Schema.Literals(["intent", "live-cleared", "markers-cleared"]) -const ResetTarget = Schema.Literals(["data", "metadata", "status", "store", "tmp"]) +const ResetTarget = Schema.Literals(["control", "data", "metadata", "status", "store", "tmp"]) const CheckpointOperationSchema = Schema.Struct({ formatVersion: Schema.Literal(OPERATION_FORMAT_VERSION), @@ -313,9 +337,23 @@ const snapshotManifestPath = (dataDir: string, checkpointId: CheckpointId): stri const snapshotBackupDir = (dataDir: string, checkpointId: CheckpointId): string => join(checkpointSnapshotDir(dataDir, checkpointId), "backup") const snapshotBackupRelativePath = (checkpointId: CheckpointId): string => `snapshots/${checkpointId}/backup` +const snapshotControlRelativePath = (checkpointId: CheckpointId): string => + `snapshots/${checkpointId}/control.sqlite` const snapshotBackupSqlPath = (checkpointId: CheckpointId): string => `backups/${snapshotBackupRelativePath(checkpointId)}` +const sha256File = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex") + +const controlValidationMatches = ( + left: EventingControlSnapshotValidation, + right: EventingControlSnapshotValidation, +): boolean => + left.schemaVersion === right.schemaVersion && + left.projectionRevisions === right.projectionRevisions && + left.projectionFailures === right.projectionFailures && + left.stagedEvents === right.stagedEvents && + left.readyEvents === right.readyEvents + const assertContained = (root: string, candidate: string, label: string): string => { const absoluteRoot = resolve(root) const absoluteCandidate = resolve(candidate) @@ -653,6 +691,12 @@ export const parseCheckpointManifest = ( if (manifest.backupRelativePath !== snapshotBackupRelativePath(manifest.checkpointId)) { throw new Error("checkpoint backup path does not match its immutable ID") } + if ( + manifest.formatVersion === MANIFEST_FORMAT_VERSION && + manifest.controlRelativePath !== snapshotControlRelativePath(manifest.checkpointId) + ) { + throw new Error("checkpoint control-store path does not match its immutable ID") + } if (manifest.chdbVersion !== CHDB_VERSION) { throw new Error( `checkpoint chDB version mismatch (checkpoint: ${manifest.chdbVersion}; build: ${CHDB_VERSION})`, @@ -763,6 +807,24 @@ const resolveCheckpointById = async ( `checkpoint backup size mismatch (manifest: ${manifest.backupBytes}; actual: ${actualBackupBytes})`, ) } + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + if (manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await assertNoSymlink(snapshotsRoot, controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlBytes = (await stat(controlPath)).size + if (controlBytes !== manifest.controlBytes) + throw new Error( + `checkpoint control-store size mismatch (manifest: ${manifest.controlBytes}; actual: ${controlBytes})`, + ) + const controlSha256 = sha256File(controlPath) + if (controlSha256 !== manifest.controlSha256) + throw new Error("checkpoint control-store digest mismatch") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + if (!controlValidationMatches(manifest.controlValidation, controlValidation)) + throw new Error("checkpoint control-store validation does not match its manifest") + } else if (existsSync(controlPath)) { + throw new Error("legacy checkpoint contains an unsigned eventing control snapshot") + } return { checkpointId, snapshotDir, @@ -804,6 +866,15 @@ const restoreResolvedInto = async ( `RESTORE DATABASE default FROM Disk('src', '${resolvedCheckpoint.backupSqlPath}') ` + "SETTINGS allow_different_database_def=1", ) + if (resolvedCheckpoint.manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await LocalEventingControlStore.restoreSnapshot( + join(resolvedCheckpoint.snapshotDir, "control.sqlite"), + targetDataDir, + ) + } else { + const controlStore = await LocalEventingControlStore.open(targetDataDir) + controlStore.close() + } return { db, validation: validateRestoredDatabase(db) } } catch (error) { db?.close() @@ -1512,10 +1583,14 @@ export const createCheckpoint = Effect.fn("CheckpointService.create")(function* const { oldState, snapshot, startedAt } = prepared let { operation } = prepared await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) + const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) operation = { ...operation, phase: "backup-complete" } await writeOperation(options.dataDir, operation, options.faults) const provisionalManifest: CheckpointManifest = { - formatVersion: 1, + formatVersion: MANIFEST_FORMAT_VERSION, checkpointId, operationId, mapleVersion: MAPLE_VERSION, @@ -1525,6 +1600,10 @@ export const createCheckpoint = Effect.fn("CheckpointService.create")(function* sourceDataDir: resolve(options.dataDir), backupRelativePath: snapshotBackupRelativePath(checkpointId), backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), + controlRelativePath: snapshotControlRelativePath(checkpointId), + controlBytes: (await stat(controlPath)).size, + controlSha256: sha256File(controlPath), + controlValidation, validation: { validatedAt: startedAt, traces: 0, @@ -1695,7 +1774,7 @@ const beginResetTransactionUnlocked = async ( const entries = await readdir(live, { withFileTypes: true }) for (const entry of entries) { if (entry.name === "backups") continue - if (!RESETTABLE_CHDB_ENTRIES.has(entry.name)) { + if (!RESETTABLE_LIVE_ENTRIES.has(entry.name)) { unknown.push(join(live, entry.name)) continue } @@ -1943,9 +2022,10 @@ export const reconcileCheckpointRecovery = Effect.fn("CheckpointService.reconcil }) /** - * Explicitly remove the live chDB store while preserving the checkpoint - * registry below `/backups`. The maintenance lock serializes this - * destructive operation with checkpoint, restore, and archive work. + * Explicitly remove the live chDB and eventing control stores while preserving + * the checkpoint registry below `/backups`. The maintenance lock + * serializes this destructive operation with checkpoint, restore, and archive + * work. */ export const resetLiveStorePreservingCheckpoints = Effect.fn("CheckpointService.reset")(function* ( dataDir: string, diff --git a/apps/cli/src/server/eventing/consumer-auth.ts b/apps/cli/src/server/eventing/consumer-auth.ts new file mode 100644 index 000000000..3751a4b5c --- /dev/null +++ b/apps/cli/src/server/eventing/consumer-auth.ts @@ -0,0 +1,35 @@ +import { randomBytes, timingSafeEqual } from "node:crypto" +import { lstatSync, readFileSync } from "node:fs" +import { resolve } from "node:path" +import { durableWrite } from "../durable-files" + +const TOKEN_BYTES = 32 + +export const eventConsumerTokenPath = (dataDir: string): string => `${resolve(dataDir)}.event-consumer-token` + +const readRealFile = (path: string): string => { + const stat = lstatSync(path) + if (stat.isSymbolicLink() || !stat.isFile()) + throw new Error(`event consumer token is not a real file: ${path}`) + return readFileSync(path, "utf8") +} + +export const ensureEventConsumerToken = async (dataDir: string): Promise => { + const path = eventConsumerTokenPath(dataDir) + try { + readRealFile(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + await durableWrite(path, `${randomBytes(TOKEN_BYTES).toString("hex")}\n`) + } + const token = readRealFile(path).trim() + if (!/^[0-9a-f]{64}$/.test(token)) throw new Error("event consumer token is malformed") + return token +} + +export const eventConsumerTokenMatches = (expected: string, supplied: string | null): boolean => { + if (supplied === null) return false + const left = Buffer.from(expected) + const right = Buffer.from(supplied) + return left.length === right.length && timingSafeEqual(left, right) +} diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts new file mode 100644 index 000000000..9f4aa5491 --- /dev/null +++ b/apps/cli/src/server/eventing/control-store.ts @@ -0,0 +1,1078 @@ +import { constants as sqliteConstants, Database } from "bun:sqlite" +import { createHash, randomBytes, timingSafeEqual } from "node:crypto" +import { chmodSync, existsSync, lstatSync, readFileSync } from "node:fs" +import { join, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { + canonicalJson, + isJsonValue, + SignalProjectionSpecSchema, + validateMapleCloudEvent, + type MapleCloudEvent, + type JsonValue, + type ProjectionFailure, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { durableWrite, ensurePrivateDirectory } from "../durable-files" +import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" + +const CONTROL_SCHEMA_VERSION = 2 +const CONTROL_DIRECTORY = "control" +const CONTROL_DATABASE = "eventing.sqlite" +const MAX_FAILURES_PER_TENANT = 10_000 +export const DEFAULT_MAX_OUTBOX_EVENTS = 10_000 +export const DEFAULT_MAX_OUTBOX_BYTES = 256 * 1024 * 1024 +export const DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS = 1_000 + +export const eventingControlDirectory = (dataDir: string): string => join(resolve(dataDir), CONTROL_DIRECTORY) +export const eventingControlPath = (dataDir: string): string => + join(eventingControlDirectory(dataDir), CONTROL_DATABASE) +export const eventingControlSnapshotPath = (dataDir: string, checkpointId: string): string => + join(resolve(dataDir), "backups", "snapshots", checkpointId, "control.sqlite") + +const CREATE_SCHEMA = ` +CREATE TABLE projection_revisions ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (revision > 0), + enabled INTEGER NOT NULL CHECK (enabled IN (0, 1)), + spec_json TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (tenant_id, projection_id, revision) +) STRICT; + +CREATE TABLE active_projections ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (tenant_id, projection_id), + FOREIGN KEY (tenant_id, projection_id, revision) + REFERENCES projection_revisions (tenant_id, projection_id, revision) + ON DELETE RESTRICT +) STRICT; + +CREATE TABLE outbox_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + state TEXT NOT NULL CHECK (state IN ('staged', 'ready')), + event_json TEXT NOT NULL, + staged_at TEXT NOT NULL, + ready_at TEXT +) STRICT; + +CREATE INDEX outbox_events_staged_sequence + ON outbox_events (state, sequence); + +CREATE TABLE outbox_ready_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + ready_at TEXT NOT NULL, + FOREIGN KEY (event_id) + REFERENCES outbox_events (event_id) + ON DELETE RESTRICT +) STRICT; + +CREATE TABLE projection_failures ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + occurrence_id TEXT, + message TEXT NOT NULL, + created_at TEXT NOT NULL +) STRICT; + +CREATE UNIQUE INDEX projection_failures_occurrence + ON projection_failures (tenant_id, projection_id, projection_revision, occurrence_id) + WHERE occurrence_id IS NOT NULL; + +CREATE TABLE event_consumers ( + consumer_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + active INTEGER NOT NULL CHECK (active IN (0, 1)), + last_acked_sequence INTEGER NOT NULL CHECK (last_acked_sequence >= 0), + lease_token_hash TEXT, + lease_expires_at TEXT, + claimed_through_sequence INTEGER CHECK (claimed_through_sequence > 0), + registered_at TEXT NOT NULL, + disabled_at TEXT, + CHECK ( + (active = 1 AND disabled_at IS NULL) OR + (active = 0 AND disabled_at IS NOT NULL) + ), + CHECK ( + (lease_token_hash IS NULL AND lease_expires_at IS NULL AND claimed_through_sequence IS NULL) OR + (lease_token_hash IS NOT NULL AND lease_expires_at IS NOT NULL AND claimed_through_sequence IS NOT NULL) + ), + CHECK (claimed_through_sequence IS NULL OR claimed_through_sequence > last_acked_sequence) +) STRICT; + +CREATE INDEX event_consumers_tenant_active_ack + ON event_consumers (tenant_id, active, last_acked_sequence); + +PRAGMA user_version = 2; +` + +const MIGRATE_SCHEMA_1_TO_2 = ` +CREATE TABLE event_consumers ( + consumer_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + active INTEGER NOT NULL CHECK (active IN (0, 1)), + last_acked_sequence INTEGER NOT NULL CHECK (last_acked_sequence >= 0), + lease_token_hash TEXT, + lease_expires_at TEXT, + claimed_through_sequence INTEGER CHECK (claimed_through_sequence > 0), + registered_at TEXT NOT NULL, + disabled_at TEXT, + CHECK ( + (active = 1 AND disabled_at IS NULL) OR + (active = 0 AND disabled_at IS NOT NULL) + ), + CHECK ( + (lease_token_hash IS NULL AND lease_expires_at IS NULL AND claimed_through_sequence IS NULL) OR + (lease_token_hash IS NOT NULL AND lease_expires_at IS NOT NULL AND claimed_through_sequence IS NOT NULL) + ), + CHECK (claimed_through_sequence IS NULL OR claimed_through_sequence > last_acked_sequence) +) STRICT; + +CREATE INDEX event_consumers_tenant_active_ack + ON event_consumers (tenant_id, active, last_acked_sequence); + +PRAGMA user_version = 2; +` + +interface UserVersionRow { + readonly user_version: number | bigint +} + +interface RevisionRow { + readonly revision: number | bigint | null +} + +interface ProjectionJsonRow { + readonly spec_json: string +} + +interface EventRow { + readonly event_id: string + readonly event_json: string + readonly state: "staged" | "ready" +} + +interface EventJsonRow { + readonly sequence: number | bigint + readonly event_json: string + readonly staged_at: string + readonly ready_at: string | null +} + +interface CountRow { + readonly count: number | bigint +} + +interface QuickCheckRow { + readonly quick_check: string +} + +interface WalCheckpointRow { + readonly busy: number | bigint + readonly log: number | bigint + readonly checkpointed: number | bigint +} + +interface OutboxUsageRow { + readonly count: number | bigint + readonly bytes: number | bigint +} + +interface SequenceRow { + readonly sequence: number | bigint | null +} + +interface ConsumerRow { + readonly consumer_id: string + readonly tenant_id: string + readonly active: number | bigint + readonly last_acked_sequence: number | bigint + readonly lease_token_hash: string | null + readonly lease_expires_at: string | null + readonly claimed_through_sequence: number | bigint | null + readonly registered_at: string + readonly disabled_at: string | null +} + +interface EventIdRow { + readonly event_id: string +} + +export interface StageEventsResult { + readonly inserted: number + readonly deduplicated: number + readonly eventIds: readonly string[] +} + +export interface EventingControlSnapshotValidation { + readonly schemaVersion: number + readonly projectionRevisions: number + readonly projectionFailures: number + readonly stagedEvents: number + readonly readyEvents: number +} + +export interface LocalEventingControlLimits { + readonly maxOutboxEvents: number + readonly maxOutboxBytes: number + readonly retainAcknowledgedReadyEvents?: number +} + +interface ResolvedLocalEventingControlLimits { + readonly maxOutboxEvents: number + readonly maxOutboxBytes: number + readonly retainAcknowledgedReadyEvents: number +} + +export interface EventingOutboxRecord { + readonly sequence: number + readonly event: MapleCloudEvent + readonly stagedAt: string + readonly readyAt: string | null +} + +export interface EventingOutboxPage { + readonly events: readonly EventingOutboxRecord[] + readonly nextCursor: number | null +} + +export type EventConsumerStart = "beginning" | "latest" + +export interface EventConsumer { + readonly consumerId: string + readonly tenantId: string + readonly active: boolean + readonly lastAcknowledgedSequence: number + readonly leaseExpiresAt: string | null + readonly claimedThroughSequence: number | null + readonly registeredAt: string + readonly disabledAt: string | null +} + +export interface EventConsumerClaim { + readonly consumerId: string + readonly leaseToken: string | null + readonly leaseExpiresAt: string | null + readonly throughSequence: number | null + readonly events: readonly EventingOutboxRecord[] +} + +export interface EventConsumerAcknowledgement { + readonly consumerId: string + readonly acknowledgedThrough: number + readonly prunedEvents: number +} + +export class EventConsumerInputError extends Error {} +export class EventConsumerNotFoundError extends Error {} +export class EventConsumerConflictError extends Error {} + +const asNumber = (value: number | bigint): number => { + const number = Number(value) + if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) + return number +} + +const decodeProjection = (json: string): SignalProjectionSpec => + Schema.decodeUnknownSync(SignalProjectionSpecSchema)(JSON.parse(json) as unknown) + +const decodeEvent = (json: string): MapleCloudEvent => { + return validateMapleCloudEvent(JSON.parse(json) as unknown).event +} + +const assertRealDatabaseFile = (path: string): void => { + let info + try { + info = lstatSync(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + if (info.isSymbolicLink() || !info.isFile()) + throw new Error(`eventing control database is not a real file: ${path}`) +} + +const configure = (db: Database): void => { + db.exec("PRAGMA foreign_keys = ON") + db.exec("PRAGMA trusted_schema = OFF") + db.exec("PRAGMA busy_timeout = 5000") +} + +const checkpointWal = (db: Database): void => { + const result = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get() + if (!result) throw new Error("eventing control WAL checkpoint returned no result") + const busy = asNumber(result.busy) + const log = asNumber(result.log) + const checkpointed = asNumber(result.checkpointed) + if (busy !== 0 || log !== 0) + throw new Error( + `eventing control WAL checkpoint incomplete (busy=${busy}, log=${log}, checkpointed=${checkpointed})`, + ) +} + +const validateLimits = (limits: LocalEventingControlLimits): ResolvedLocalEventingControlLimits => { + if (!Number.isSafeInteger(limits.maxOutboxEvents) || limits.maxOutboxEvents < 1) + throw new Error("maxOutboxEvents must be a positive safe integer") + if (!Number.isSafeInteger(limits.maxOutboxBytes) || limits.maxOutboxBytes < 1) + throw new Error("maxOutboxBytes must be a positive safe integer") + const retainAcknowledgedReadyEvents = + limits.retainAcknowledgedReadyEvents ?? DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS + if (!Number.isSafeInteger(retainAcknowledgedReadyEvents) || retainAcknowledgedReadyEvents < 0) + throw new Error("retainAcknowledgedReadyEvents must be a non-negative safe integer") + return { ...limits, retainAcknowledgedReadyEvents } +} + +const validateOpenDatabase = ( + db: Database, + acceptedSchemaVersions: readonly number[] = [CONTROL_SCHEMA_VERSION], +): EventingControlSnapshotValidation => { + const quick = db.query("PRAGMA quick_check").get() + if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + const schemaVersion = asNumber(version.user_version) + if (!acceptedSchemaVersions.includes(schemaVersion)) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${acceptedSchemaVersions.join(" or ")}`, + ) + const count = (where: string): number => { + const row = db.query(`SELECT count(*) AS count FROM outbox_events ${where}`).get() + if (!row) throw new Error("eventing control count query returned no row") + return asNumber(row.count) + } + const revisions = db.query("SELECT count(*) AS count FROM projection_revisions").get() + if (!revisions) throw new Error("eventing projection count query returned no row") + const failures = db.query("SELECT count(*) AS count FROM projection_failures").get() + if (!failures) throw new Error("eventing projection-failure count query returned no row") + const invalidReadiness = db + .query( + `SELECT count(*) AS count + FROM outbox_events AS event + LEFT JOIN outbox_ready_events AS readiness ON readiness.event_id = event.event_id + WHERE (event.state = 'ready' AND ( + readiness.event_id IS NULL OR event.ready_at IS NULL OR event.ready_at <> readiness.ready_at + )) OR (event.state = 'staged' AND ( + readiness.event_id IS NOT NULL OR event.ready_at IS NOT NULL + ))`, + ) + .get() + if (!invalidReadiness) throw new Error("eventing readiness validation query returned no row") + if (asNumber(invalidReadiness.count) !== 0) + throw new Error("eventing control database has inconsistent outbox readiness state") + if (schemaVersion >= 2) { + const consumers = db + .query, []>( + "SELECT lease_expires_at, registered_at, disabled_at FROM event_consumers", + ) + .all() + for (const consumer of consumers) { + canonicalInstant(consumer.registered_at, "event consumer registeredAt") + if (consumer.lease_expires_at !== null) + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") + if (consumer.disabled_at !== null) + canonicalInstant(consumer.disabled_at, "event consumer disabledAt") + } + } + return { + schemaVersion, + projectionRevisions: asNumber(revisions.count), + projectionFailures: asNumber(failures.count), + stagedEvents: count("WHERE state = 'staged'"), + readyEvents: count("WHERE state = 'ready'"), + } +} + +const CONSUMER_ID = /^[a-z][a-z0-9._-]{0,63}$/ +const LEASE_TOKEN = /^[0-9a-f]{64}$/ + +const validateConsumerId = (consumerId: string): string => { + if (!CONSUMER_ID.test(consumerId)) + throw new EventConsumerInputError( + "consumerId must start with a lowercase letter and contain at most 64 lowercase letters, digits, dots, underscores, or hyphens", + ) + return consumerId +} + +const canonicalInstant = (value: string, label: string): number => { + const milliseconds = Date.parse(value) + if (Number.isNaN(milliseconds) || new Date(milliseconds).toISOString() !== value) + throw new EventConsumerInputError(`${label} must be canonical ISO-8601`) + return milliseconds +} + +const tokenHash = (token: string): string => createHash("sha256").update(token).digest("hex") + +const tokenHashMatches = (expected: string, token: string): boolean => { + if (!LEASE_TOKEN.test(token)) return false + const left = Buffer.from(expected, "hex") + const right = Buffer.from(tokenHash(token), "hex") + return left.length === right.length && timingSafeEqual(left, right) +} + +const decodeConsumer = (row: ConsumerRow): EventConsumer => ({ + consumerId: row.consumer_id, + tenantId: row.tenant_id, + active: asNumber(row.active) === 1, + lastAcknowledgedSequence: asNumber(row.last_acked_sequence), + leaseExpiresAt: row.lease_expires_at, + claimedThroughSequence: + row.claimed_through_sequence === null ? null : asNumber(row.claimed_through_sequence), + registeredAt: row.registered_at, + disabledAt: row.disabled_at, +}) + +export class LocalEventingControlStore { + readonly #db: Database + readonly #limits: ResolvedLocalEventingControlLimits + readonly #telemetry: EventingTelemetry + readonly path: string + + private constructor( + path: string, + db: Database, + limits: ResolvedLocalEventingControlLimits, + telemetry: EventingTelemetry, + ) { + this.path = path + this.#db = db + this.#limits = limits + this.#telemetry = telemetry + } + + static async open( + dataDir: string, + limits: LocalEventingControlLimits = { + maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, + maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, + retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, + }, + telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, + ): Promise { + const validatedLimits = validateLimits(limits) + const directory = eventingControlDirectory(dataDir) + await ensurePrivateDirectory(directory) + const path = eventingControlPath(dataDir) + assertRealDatabaseFile(path) + const db = new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) + try { + configure(db) + db.exec("PRAGMA journal_mode = WAL") + db.exec("PRAGMA synchronous = FULL") + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + const schemaVersion = asNumber(version.user_version) + if (schemaVersion === 0) db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() + else if (schemaVersion === 1) db.transaction(() => db.exec(MIGRATE_SCHEMA_1_TO_2)).exclusive() + else if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + chmodSync(path, 0o600) + validateOpenDatabase(db) + return new LocalEventingControlStore(path, db, validatedLimits, telemetry) + } catch (error) { + db.close() + throw error + } + } + + close(): void { + checkpointWal(this.#db) + this.#db.close(true) + } + + saveProjection(spec: SignalProjectionSpec, createdAt = new Date().toISOString()): void { + const decoded = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(spec) + if (!isJsonValue(decoded as unknown)) throw new Error("projection spec must be finite JSON") + const specJson = canonicalJson(decoded as unknown as JsonValue) + this.#db + .transaction(() => { + const existing = this.#db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(decoded.tenantId, decoded.id, decoded.revision) + if (existing) { + if (existing.spec_json !== specJson) + throw new Error( + `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + } else { + const latest = this.#db + .query( + "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const expected = latest?.revision == null ? 1 : asNumber(latest.revision) + 1 + if (decoded.revision !== expected) + throw new Error( + `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + this.#db.run( + "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + decoded.tenantId, + decoded.id, + decoded.revision, + decoded.enabled ? 1 : 0, + specJson, + createdAt, + ], + ) + } + + if (decoded.enabled) + this.#db.run( + "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", + [decoded.tenantId, decoded.id, decoded.revision], + ) + else + this.#db.run("DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", [ + decoded.tenantId, + decoded.id, + ]) + }) + .immediate() + } + + loadEnabledProjections(tenantId: string): readonly SignalProjectionSpec[] { + return this.#db + .query( + `SELECT r.spec_json + FROM active_projections a + JOIN projection_revisions r + ON r.tenant_id = a.tenant_id + AND r.projection_id = a.projection_id + AND r.revision = a.revision + WHERE a.tenant_id = ? + ORDER BY a.projection_id`, + ) + .all(tenantId) + .map(({ spec_json }) => decodeProjection(spec_json)) + } + + stageEvents(events: readonly MapleCloudEvent[], stagedAt = new Date().toISOString()): StageEventsResult { + let inserted = 0 + let deduplicated = 0 + const eventIds: string[] = [] + try { + this.#db + .transaction(() => { + const usage = this.#db + .query( + "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", + ) + .get() + if (!usage) throw new Error("event outbox usage query returned no row") + let outboxEvents = asNumber(usage.count) + let outboxBytes = asNumber(usage.bytes) + for (const candidate of events) { + const validated = validateMapleCloudEvent(candidate) + const { event, canonicalJson: eventJson, byteLength: eventBytes } = validated + const existing = this.#db + .query( + "SELECT event_id, event_json, state FROM outbox_events WHERE event_id = ?", + ) + .get(event.id) + if (existing) { + if (existing.event_json !== eventJson) + throw new Error(`event ID collision with different payload: ${event.id}`) + deduplicated += 1 + } else { + if ( + outboxEvents + 1 > this.#limits.maxOutboxEvents || + outboxBytes + eventBytes > this.#limits.maxOutboxBytes + ) + throw new Error( + `event outbox capacity exceeded (${outboxEvents}/${this.#limits.maxOutboxEvents} events, ${outboxBytes}/${this.#limits.maxOutboxBytes} bytes)`, + ) + this.#db.run( + "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, state, event_json, staged_at) VALUES (?, ?, ?, ?, 'staged', ?, ?)", + [ + event.id, + event.tenantid, + event.projectionid, + event.projectionrevision, + eventJson, + stagedAt, + ], + ) + inserted += 1 + outboxEvents += 1 + outboxBytes += eventBytes + } + eventIds.push(event.id) + } + }) + .immediate() + } catch (error) { + this.#telemetry.record({ operation: "outbox_stage", outcome: "failure" }) + throw error + } + this.#telemetry.record({ operation: "outbox_stage", outcome: "success", count: inserted }) + this.#telemetry.record({ operation: "outbox_dedup", outcome: "success", count: deduplicated }) + return { inserted, deduplicated, eventIds } + } + + markReady(eventIds: readonly string[], readyAt = new Date().toISOString()): void { + let markedReady = 0 + try { + this.#db + .transaction(() => { + for (const eventId of eventIds) { + const row = this.#db + .query, [string]>( + "SELECT state FROM outbox_events WHERE event_id = ?", + ) + .get(eventId) + if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) + if (row.state === "ready") continue + this.#db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ + eventId, + readyAt, + ]) + this.#db.run( + "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", + [readyAt, eventId], + ) + markedReady += 1 + } + }) + .immediate() + } catch (error) { + this.#telemetry.record({ operation: "outbox_ready", outcome: "failure" }) + throw error + } + this.#telemetry.record({ operation: "outbox_ready", outcome: "success", count: markedReady }) + } + + #listOutbox(state: "ready" | "staged", limit = 100, after = 0): EventingOutboxPage { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new Error("outbox-event limit must be between 1 and 1000") + if (!Number.isSafeInteger(after) || after < 0) + throw new Error("outbox cursor must be a non-negative safe integer") + const rows = + state === "ready" + ? this.#db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(after, limit + 1) + : this.#db + .query( + `SELECT sequence, event_json, staged_at, ready_at + FROM outbox_events + WHERE state = 'staged' AND sequence > ? + ORDER BY sequence + LIMIT ?`, + ) + .all(after, limit + 1) + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })) + return { + events: page, + nextCursor: hasMore ? (page.at(-1)?.sequence ?? null) : null, + } + } + + listReady(limit = 100, after = 0): EventingOutboxPage { + return this.#listOutbox("ready", limit, after) + } + + listStaged(limit = 100, after = 0): EventingOutboxPage { + return this.#listOutbox("staged", limit, after) + } + + listConsumers(tenantId: string): readonly EventConsumer[] { + return this.#db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? + ORDER BY consumer_id`, + ) + .all(tenantId) + .map(decodeConsumer) + } + + registerConsumer( + tenantId: string, + consumerId: string, + startAt: EventConsumerStart, + registeredAt = new Date().toISOString(), + ): EventConsumer { + validateConsumerId(consumerId) + if (startAt !== "beginning" && startAt !== "latest") + throw new EventConsumerInputError("startAt must be beginning or latest") + canonicalInstant(registeredAt, "event consumer registeredAt") + return this.#db + .transaction(() => { + const existing = this.#consumer(tenantId, consumerId) + if (existing) + throw new EventConsumerConflictError(`event consumer already exists: ${consumerId}`) + const boundary = this.#db + .query( + startAt === "latest" + ? `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ?` + : `SELECT min(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ?`, + ) + .get(tenantId) + const sequence = boundary?.sequence == null ? 0 : asNumber(boundary.sequence) + const lastAcknowledged = startAt === "beginning" ? Math.max(0, sequence - 1) : sequence + this.#db.run( + "INSERT INTO event_consumers (consumer_id, tenant_id, active, last_acked_sequence, registered_at) VALUES (?, ?, 1, ?, ?)", + [consumerId, tenantId, lastAcknowledged, registeredAt], + ) + return decodeConsumer(this.#consumer(tenantId, consumerId)!) + }) + .immediate() + } + + disableConsumer( + tenantId: string, + consumerId: string, + disabledAt = new Date().toISOString(), + ): EventConsumer { + validateConsumerId(consumerId) + canonicalInstant(disabledAt, "event consumer disabledAt") + return this.#db + .transaction(() => { + const existing = this.#consumer(tenantId, consumerId) + if (!existing) throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(existing.active) === 0) return decodeConsumer(existing) + this.#db.run( + `UPDATE event_consumers + SET active = 0, lease_token_hash = NULL, lease_expires_at = NULL, + claimed_through_sequence = NULL, disabled_at = ? + WHERE tenant_id = ? AND consumer_id = ?`, + [disabledAt, tenantId, consumerId], + ) + this.#pruneAcknowledgedReady(tenantId) + return decodeConsumer(this.#consumer(tenantId, consumerId)!) + }) + .immediate() + } + + claimReady( + tenantId: string, + consumerId: string, + limit: number, + leaseSeconds: number, + now = new Date().toISOString(), + ): EventConsumerClaim { + let reclaimedExpiredLease = false + let lag = 0 + try { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new EventConsumerInputError("claim limit must be between 1 and 1000") + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) + throw new EventConsumerInputError("leaseSeconds must be between 5 and 300") + const nowMilliseconds = canonicalInstant(now, "claim time") + const claim = this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) + throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_expires_at !== null && + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > + nowMilliseconds + ) + throw new EventConsumerConflictError( + `event consumer already has an active lease: ${consumerId}`, + ) + if (consumer.lease_expires_at !== null) reclaimedExpiredLease = true + lag = this.#consumerLag(tenantId, asNumber(consumer.last_acked_sequence)) + + const rows = this.#db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(tenantId, asNumber(consumer.last_acked_sequence), limit) + if (rows.length === 0) { + this.#db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", + [tenantId, consumerId], + ) + return { + consumerId, + leaseToken: null, + leaseExpiresAt: null, + throughSequence: null, + events: [], + } + } + + const leaseToken = randomBytes(32).toString("hex") + const leaseExpiresAt = new Date(nowMilliseconds + leaseSeconds * 1_000).toISOString() + const throughSequence = asNumber(rows.at(-1)!.sequence) + this.#db.run( + `UPDATE event_consumers + SET lease_token_hash = ?, lease_expires_at = ?, claimed_through_sequence = ? + WHERE tenant_id = ? AND consumer_id = ?`, + [tokenHash(leaseToken), leaseExpiresAt, throughSequence, tenantId, consumerId], + ) + return { + consumerId, + leaseToken, + leaseExpiresAt, + throughSequence, + events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })), + } + }) + .immediate() + this.#telemetry.record({ + operation: "consumer_claim", + outcome: claim.events.length === 0 ? "empty" : "success", + count: Math.max(1, claim.events.length), + }) + this.#telemetry.record({ operation: "consumer_lag", outcome: "observed", lag }) + if (reclaimedExpiredLease) + this.#telemetry.record({ operation: "consumer_lease", outcome: "reclaimed" }) + return claim + } catch (error) { + this.#telemetry.record({ operation: "consumer_claim", outcome: "failure" }) + if (error instanceof EventConsumerConflictError && /lease/.test(error.message)) + this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) + throw error + } + } + + acknowledgeClaim( + tenantId: string, + consumerId: string, + leaseToken: string, + throughSequence: number, + now = new Date().toISOString(), + ): EventConsumerAcknowledgement { + try { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) + throw new EventConsumerInputError("throughSequence must be a positive safe integer") + const nowMilliseconds = canonicalInstant(now, "acknowledgement time") + const acknowledgement = this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) + throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_token_hash === null || + consumer.lease_expires_at === null || + consumer.claimed_through_sequence === null + ) + throw new EventConsumerConflictError( + `event consumer has no active lease: ${consumerId}`, + ) + if ( + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") <= + nowMilliseconds + ) + throw new EventConsumerConflictError( + `event consumer lease has expired: ${consumerId}`, + ) + if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) + throw new EventConsumerConflictError("event consumer lease token does not match") + const claimedThrough = asNumber(consumer.claimed_through_sequence) + if (throughSequence !== claimedThrough) + throw new EventConsumerConflictError( + `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, + ) + this.#db.run( + `UPDATE event_consumers + SET last_acked_sequence = ?, lease_token_hash = NULL, lease_expires_at = NULL, + claimed_through_sequence = NULL + WHERE tenant_id = ? AND consumer_id = ?`, + [throughSequence, tenantId, consumerId], + ) + return { + consumerId, + acknowledgedThrough: throughSequence, + prunedEvents: this.#pruneAcknowledgedReady(tenantId), + } + }) + .immediate() + this.#telemetry.record({ operation: "consumer_ack", outcome: "success" }) + this.#telemetry.record({ + operation: "consumer_lag", + outcome: "observed", + lag: this.#consumerLag(tenantId, acknowledgement.acknowledgedThrough), + }) + return acknowledgement + } catch (error) { + this.#telemetry.record({ operation: "consumer_ack", outcome: "failure" }) + if (error instanceof EventConsumerConflictError && /lease/.test(error.message)) + this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) + throw error + } + } + + #consumerLag(tenantId: string, lastAcknowledgedSequence: number): number { + const latest = this.#db + .query( + `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready'`, + ) + .get(tenantId) + return Math.max( + 0, + (latest?.sequence == null ? 0 : asNumber(latest.sequence)) - lastAcknowledgedSequence, + ) + } + + #consumer(tenantId: string, consumerId: string): ConsumerRow | null { + return this.#db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? AND consumer_id = ?`, + ) + .get(tenantId, consumerId) + } + + #pruneAcknowledgedReady(tenantId: string): number { + const boundary = this.#db + .query( + "SELECT min(last_acked_sequence) AS sequence FROM event_consumers WHERE tenant_id = ? AND active = 1", + ) + .get(tenantId) + if (boundary?.sequence == null) return 0 + const rows = this.#db + .query( + `SELECT readiness.event_id + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND readiness.sequence <= ? + ORDER BY readiness.sequence`, + ) + .all(tenantId, asNumber(boundary.sequence)) + const pruneCount = Math.max(0, rows.length - this.#limits.retainAcknowledgedReadyEvents) + for (const { event_id } of rows.slice(0, pruneCount)) { + this.#db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [event_id]) + this.#db.run("DELETE FROM outbox_events WHERE event_id = ? AND state = 'ready'", [event_id]) + } + return pruneCount + } + + outboxCapacity(): LocalEventingControlLimits & { + readonly currentEvents: number + readonly currentBytes: number + } { + const usage = this.#db + .query( + "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", + ) + .get() + if (!usage) throw new Error("event outbox usage query returned no row") + return { + ...this.#limits, + currentEvents: asNumber(usage.count), + currentBytes: asNumber(usage.bytes), + } + } + + recordProjectionFailures( + tenantId: string, + failures: readonly ProjectionFailure[], + createdAt = new Date().toISOString(), + ): void { + this.#db + .transaction(() => { + for (const failure of failures) + this.#db.run( + "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + tenantId, + failure.projectionId, + failure.projectionRevision, + failure.occurrenceId, + failure.message.slice(0, 4_096), + createdAt, + ], + ) + this.#db.run( + "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", + [tenantId, tenantId, MAX_FAILURES_PER_TENANT], + ) + }) + .immediate() + } + + validate(): EventingControlSnapshotValidation { + return validateOpenDatabase(this.#db) + } + + async backupTo(path: string): Promise { + // sqlite3_serialize() snapshots the main database file. In WAL mode a + // committed transaction may still live only in the sidecar, so force and + // verify a complete checkpoint before copying the file image. + checkpointWal(this.#db) + const bytes = this.#db.serialize() + await durableWrite(path, bytes) + return LocalEventingControlStore.validateSnapshot(path) + } + + static validateSnapshot(path: string): EventingControlSnapshotValidation { + assertRealDatabaseFile(path) + if (!existsSync(path)) throw new Error(`eventing control snapshot is missing: ${path}`) + const uri = `${pathToFileURL(path).href}?immutable=1` + const db = new Database(uri, sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI) + try { + configure(db) + return validateOpenDatabase(db, [1, CONTROL_SCHEMA_VERSION]) + } finally { + db.close(true) + } + } + + static async restoreSnapshot(snapshotPath: string, dataDir: string): Promise { + LocalEventingControlStore.validateSnapshot(snapshotPath) + await durableWrite(eventingControlPath(dataDir), readFileSync(snapshotPath)) + } +} diff --git a/apps/cli/src/server/eventing/gitlab-projectors.ts b/apps/cli/src/server/eventing/gitlab-projectors.ts new file mode 100644 index 000000000..c1ee2a9f8 --- /dev/null +++ b/apps/cli/src/server/eventing/gitlab-projectors.ts @@ -0,0 +1,1053 @@ +import { + ProjectorRegistry, + fieldKey, + isJsonValue, + type NormalizedSignal, + type SignalScalar, +} from "@maple/eventing-core" + +export interface GitLabIssueProjectorConfig { + readonly includeBody: boolean +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const gitlabProjectorConfig = (value: unknown): GitLabIssueProjectorConfig => { + if (!isRecord(value)) throw new Error("gitlab.issue.created projector config must be an object") + const keys = Object.keys(value) + if (keys.some((key) => key !== "includeBody")) + throw new Error("gitlab.issue.created projector config contains an unknown field") + if (value.includeBody !== undefined && typeof value.includeBody !== "boolean") + throw new Error("gitlab.issue.created includeBody must be boolean") + return { includeBody: value.includeBody === true } +} + +const field = (signal: NormalizedSignal, namespace: "signal" | "resource" | "attribute", key: string) => + signal.fields.get(fieldKey({ namespace, key })) + +const scalarString = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "string") throw new Error(`GitLab issue event ${label} must be a string`) + return value.value +} + +const scalarInt64 = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "int64") throw new Error(`GitLab issue event ${label} must be an int64`) + return value.value +} + +export const gitlabIssueCreatedProjector = { + id: "gitlab.issue.created", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.created.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-created:v1", + decodeConfig: gitlabProjectorConfig, + project: (signal: NormalizedSignal, config: GitLabIssueProjectorConfig) => { + const projectId = scalarInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") + const projectPath = scalarString( + field(signal, "attribute", "gitlab.project.path"), + "gitlab.project.path", + true, + )! + const issueId = scalarInt64(field(signal, "attribute", "gitlab.issue.id"), "gitlab.issue.id") + const issueIid = scalarInt64( + field(signal, "attribute", "gitlab.issue.iid"), + "gitlab.issue.iid", + true, + )! + const title = scalarString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") + const url = scalarString(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") + const actorId = scalarInt64(field(signal, "attribute", "gitlab.user.id"), "gitlab.user.id") + const actorUsername = scalarString( + field(signal, "attribute", "gitlab.user.username"), + "gitlab.user.username", + ) + const serviceName = scalarString(field(signal, "resource", "service.name"), "service.name") + const candidateBody = + isRecord(signal.data) && isRecord(signal.data.record) ? signal.data.record.body : undefined + const body = isJsonValue(candidateBody) ? candidateBody : undefined + return { + subject: `${projectPath}/issues/${issueIid}`, + data: { + project: { + ...(projectId === undefined ? {} : { id: projectId }), + path: projectPath, + }, + issue: { + ...(issueId === undefined ? {} : { id: issueId }), + iid: issueIid, + ...(title === undefined ? {} : { title }), + ...(url === undefined ? {} : { url }), + }, + ...(actorId === undefined && actorUsername === undefined + ? {} + : { + actor: { + ...(actorId === undefined ? {} : { id: actorId }), + ...(actorUsername === undefined ? {} : { username: actorUsername }), + }, + }), + ...(serviceName === undefined ? {} : { serviceName }), + ...(config.includeBody && body !== undefined ? { body } : {}), + }, + } + }, +} as const + +type GitLabProjectAction = + | "created" + | "destroyed" + | "renamed" + | "transferred" + | "updated" + | "archived" + | "unarchived" + | "deletion_requested" + +const PROJECT_LIFECYCLE_ACTIONS: Readonly> = { + project_create: "created", + project_destroy: "destroyed", + project_rename: "renamed", + project_transfer: "transferred", + project_update: "updated", + project_archive: "archived", + project_unarchive: "unarchived", + project_deletion_request: "deletion_requested", +} + +const TERMINAL_PIPELINE_STATUSES = new Set(["success", "failed", "canceled", "skipped"]) +const ISSUE_LIFECYCLE_ACTIONS = { + issue_open: "open", + issue_update: "update", + issue_close: "close", + issue_reopen: "reopen", +} as const +const MERGE_REQUEST_LIFECYCLE_ACTIONS = { + merge_request_open: "open", + merge_request_update: "update", + merge_request_close: "close", + merge_request_reopen: "reopen", + merge_request_merge: "merge", + merge_request_review: "review", +} as const +const DEPLOYMENT_STATUSES = { + deployment_running: "running", + deployment_success: "success", + deployment_failed: "failed", + deployment_canceled: "canceled", + deployment_blocked: "blocked", + deployment_manual: "manual", +} as const +const JOB_STATUSES = new Set([ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled", +]) +const RELEASE_ACTIONS = new Set(["create", "update", "delete"]) +const MAX_PROJECTOR_TEXT_BYTES = 4 * 1024 +const MAX_COMMENT_EXCERPT_BYTES = 1024 +const MAX_CANONICAL_URL_BYTES = 2048 +const MAX_FAILED_JOBS = 20 +const MAX_ISSUE_LABELS = 50 +const MAX_ISSUE_LABEL_BYTES = 256 + +const projectorString = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + if (value === undefined) { + if (required) throw new Error(`GitLab projector is missing ${label}`) + return undefined + } + if (value.type !== "string") throw new Error(`GitLab projector ${label} must be a string`) + const text = value.value.trim() + if (text.length === 0) throw new Error(`GitLab projector ${label} must not be blank`) + if (Buffer.byteLength(text, "utf8") > MAX_PROJECTOR_TEXT_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_PROJECTOR_TEXT_BYTES} UTF-8 bytes`) + return text +} + +const projectorToken = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + const token = projectorString(value, label, required) + if (token !== undefined && !/^[a-z][a-z0-9_]{0,63}$/.test(token)) + throw new Error(`GitLab projector ${label} must be a lowercase token`) + return token +} + +const projectorUrl = ( + value: SignalScalar | undefined, + label: string, + required = false, + allowFragment = false, +): string | undefined => { + const text = projectorString(value, label, required) + if (text === undefined) return undefined + if (Buffer.byteLength(text, "utf8") > MAX_CANONICAL_URL_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_CANONICAL_URL_BYTES} UTF-8 bytes`) + let parsed: URL + try { + parsed = new URL(text) + } catch { + throw new Error(`GitLab projector ${label} must be an absolute URL`) + } + if ((parsed.protocol !== "https:" && parsed.protocol !== "http:") || parsed.username || parsed.password) + throw new Error(`GitLab projector ${label} must be an HTTP(S) URL without credentials`) + if (parsed.search || (!allowFragment && parsed.hash)) + throw new Error(`GitLab projector ${label} must not contain a query or fragment`) + return text +} + +const projectorRevision = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + const revision = projectorString(value, label, required) + if (revision !== undefined && !/^[0-9a-f]{6,64}$/i.test(revision)) + throw new Error(`GitLab projector ${label} must be a hexadecimal revision`) + return revision +} + +const projectorBoolean = (value: SignalScalar | undefined, label: string): boolean | undefined => { + if (value === undefined) return undefined + if (value.type !== "boolean") throw new Error(`GitLab projector ${label} must be a boolean`) + return value.value +} + +const projectorExcerpt = (value: SignalScalar | undefined, label: string): string | undefined => { + const source = projectorString(value, label) + if (source === undefined) return undefined + const excerpt = source + .replace(/[\p{Cc}]/gu, " ") + .replace(/\s+/gu, " ") + .trim() + if (excerpt.length === 0) throw new Error(`GitLab projector ${label} must not be blank after sanitizing`) + if (Buffer.byteLength(excerpt, "utf8") > MAX_COMMENT_EXCERPT_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_COMMENT_EXCERPT_BYTES} UTF-8 bytes`) + return excerpt +} + +const projectorInt64 = ( + value: SignalScalar | undefined, + label: string, + required = false, +): string | undefined => { + if (value === undefined) { + if (required) throw new Error(`GitLab projector is missing ${label}`) + return undefined + } + if (value.type !== "int64") throw new Error(`GitLab projector ${label} must be an int64`) + return value.value +} + +const positiveProjectorInt64 = (value: SignalScalar | undefined, label: string): string => { + const parsed = projectorInt64(value, label, true)! + if (BigInt(parsed) <= 0n) throw new Error(`GitLab projector ${label} must be positive`) + return parsed +} + +const optionalPositiveProjectorInt64 = ( + value: SignalScalar | undefined, + label: string, +): string | undefined => { + const parsed = projectorInt64(value, label) + if (parsed !== undefined && BigInt(parsed) <= 0n) + throw new Error(`GitLab projector ${label} must be positive`) + return parsed +} + +const nonNegativeProjectorInt64 = (value: SignalScalar | undefined, label: string): string | undefined => { + const parsed = projectorInt64(value, label) + if (parsed !== undefined && BigInt(parsed) < 0n) + throw new Error(`GitLab projector ${label} must not be negative`) + return parsed +} + +const eventName = (signal: NormalizedSignal): string => + projectorString( + field(signal, "attribute", "event.name") ?? field(signal, "signal", "event.name"), + "event.name", + true, + )! + +const requireSourceProvidedIdentity = (signal: NormalizedSignal): void => { + if (signal.identityQuality !== "source" || signal.occurrenceId === null) + throw new Error("GitLab projector requires source-provided occurrence identity") +} + +const actor = (signal: NormalizedSignal) => { + const id = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.actor.id"), + "gitlab.actor.id", + ) + const name = projectorString(field(signal, "attribute", "gitlab.actor.name"), "gitlab.actor.name") + const username = projectorString( + field(signal, "attribute", "gitlab.actor.username"), + "gitlab.actor.username", + ) + if (name === undefined && username === undefined && id === undefined) return undefined + return { + ...(id === undefined ? {} : { id }), + ...(name === undefined ? {} : { name }), + ...(username === undefined ? {} : { username }), + } +} + +const issue = (signal: NormalizedSignal) => { + const id = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.issue.id"), + "gitlab.issue.id", + ) + const iid = positiveProjectorInt64(field(signal, "attribute", "gitlab.issue.iid"), "gitlab.issue.iid") + const title = projectorString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") + const url = projectorUrl(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") + const state = projectorToken(field(signal, "attribute", "gitlab.issue.state"), "gitlab.issue.state") + const type = projectorToken(field(signal, "attribute", "gitlab.issue.type"), "gitlab.issue.type", true)! + const labels = issueLabels(signal) + return { + ...(id === undefined ? {} : { id }), + iid, + type, + ...(title === undefined ? {} : { title }), + ...(url === undefined ? {} : { url }), + ...(state === undefined ? {} : { state }), + ...(labels === undefined ? {} : { labels }), + } +} + +const mergeRequest = (signal: NormalizedSignal) => { + const iid = positiveProjectorInt64( + field(signal, "attribute", "gitlab.merge_request.iid"), + "gitlab.merge_request.iid", + ) + const title = projectorString( + field(signal, "attribute", "gitlab.merge_request.title"), + "gitlab.merge_request.title", + true, + )! + const url = projectorUrl( + field(signal, "attribute", "gitlab.merge_request.url"), + "gitlab.merge_request.url", + true, + )! + const sourceBranch = projectorString( + field(signal, "attribute", "gitlab.merge_request.source_branch"), + "gitlab.merge_request.source_branch", + ) + const targetBranch = projectorString( + field(signal, "attribute", "gitlab.merge_request.target_branch"), + "gitlab.merge_request.target_branch", + ) + const commit = projectorRevision( + field(signal, "attribute", "gitlab.merge_request.commit"), + "gitlab.merge_request.commit", + ) + const reviewState = projectorToken( + field(signal, "attribute", "gitlab.merge_request.review_state"), + "gitlab.merge_request.review_state", + ) + return { + iid, + title, + url, + ...(sourceBranch === undefined ? {} : { sourceBranch }), + ...(targetBranch === undefined ? {} : { targetBranch }), + ...(commit === undefined ? {} : { commit }), + ...(reviewState === undefined ? {} : { reviewState }), + } +} + +const comment = (signal: NormalizedSignal, kind?: "comment" | "review") => { + const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.comment.id"), "gitlab.comment.id") + const excerpt = projectorExcerpt( + field(signal, "attribute", "gitlab.comment.excerpt"), + "gitlab.comment.excerpt", + ) + if (excerpt === undefined) throw new Error("GitLab projector is missing gitlab.comment.excerpt") + const url = projectorUrl( + field(signal, "attribute", "gitlab.comment.url"), + "gitlab.comment.url", + true, + true, + )! + const system = projectorBoolean( + field(signal, "attribute", "gitlab.comment.system"), + "gitlab.comment.system", + ) + return { + id, + ...(kind === undefined ? {} : { kind }), + excerpt, + url, + ...(system === undefined ? {} : { system }), + } +} + +const structuredAttribute = (signal: NormalizedSignal, key: string): unknown => { + if (!isRecord(signal.data) || !isRecord(signal.data.record) || !isRecord(signal.data.record.attributes)) + return undefined + return signal.data.record.attributes[key] +} + +const issueLabels = (signal: NormalizedSignal): readonly string[] | undefined => { + const value = structuredAttribute(signal, "gitlab.issue.labels") + if (value === undefined) return undefined + if (!Array.isArray(value)) throw new Error("GitLab projector gitlab.issue.labels must be an array") + if (value.length > MAX_ISSUE_LABELS) + throw new Error(`GitLab projector gitlab.issue.labels exceeds ${MAX_ISSUE_LABELS} labels`) + return value.map((candidate, index) => { + const label = `gitlab.issue.labels[${index}]` + if (typeof candidate !== "string") throw new Error(`GitLab projector ${label} must be a string`) + const normalized = candidate.trim() + if (normalized.length === 0) throw new Error(`GitLab projector ${label} must not be blank`) + if (Buffer.byteLength(normalized, "utf8") > MAX_ISSUE_LABEL_BYTES) + throw new Error(`GitLab projector ${label} exceeds ${MAX_ISSUE_LABEL_BYTES} UTF-8 bytes`) + return normalized + }) +} + +const failedJobs = (signal: NormalizedSignal) => { + const value = structuredAttribute(signal, "gitlab.ci.pipeline.failed_jobs") + if (value === undefined) return undefined + if (!Array.isArray(value)) + throw new Error("GitLab projector gitlab.ci.pipeline.failed_jobs must be an array") + if (value.length > MAX_FAILED_JOBS) + throw new Error(`GitLab projector gitlab.ci.pipeline.failed_jobs exceeds ${MAX_FAILED_JOBS} jobs`) + return value.map((candidate, index) => { + const label = `gitlab.ci.pipeline.failed_jobs[${index}]` + if (!isRecord(candidate)) throw new Error(`GitLab projector ${label} must be an object`) + const allowed = new Set(["id", "name", "stage", "status", "url"]) + if (Object.keys(candidate).some((key) => !allowed.has(key))) + throw new Error(`GitLab projector ${label} contains an unknown field`) + const scalar = (key: string): SignalScalar | undefined => { + const item = candidate[key] + if (item === undefined) return undefined + if (typeof item !== "string") throw new Error(`GitLab projector ${label}.${key} must be a string`) + if (key === "id") return { type: "int64", value: item } + return { type: "string", value: item } + } + const id = positiveProjectorInt64(scalar("id"), `${label}.id`) + const name = projectorString(scalar("name"), `${label}.name`, true)! + const stage = projectorString(scalar("stage"), `${label}.stage`) + const status = projectorToken(scalar("status"), `${label}.status`, true)! + if (status !== "failed") throw new Error(`GitLab projector ${label}.status must be failed`) + const url = projectorUrl(scalar("url"), `${label}.url`) + return { + id, + name, + ...(stage === undefined ? {} : { stage }), + status, + ...(url === undefined ? {} : { url }), + } + }) +} + +const project = (signal: NormalizedSignal) => { + requireSourceProvidedIdentity(signal) + const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") + const path = projectorString( + field(signal, "attribute", "gitlab.project.path"), + "gitlab.project.path", + true, + )! + const oldPath = projectorString( + field(signal, "attribute", "gitlab.project.old_path"), + "gitlab.project.old_path", + ) + return { + id, + path, + ...(oldPath === undefined ? {} : { oldPath }), + } +} + +const serviceName = (signal: NormalizedSignal): string | undefined => + projectorString(field(signal, "resource", "service.name"), "service.name") + +const result = (signal: NormalizedSignal): string | undefined => + projectorString(field(signal, "attribute", "gitlab.event.result"), "gitlab.event.result") + +const noProjectorConfig = + (projectorId: string) => + (value: unknown): Record => { + if (!isRecord(value)) throw new Error(`${projectorId} projector config must be an object`) + if (Object.keys(value).length > 0) + throw new Error(`${projectorId} projector config contains unknown fields`) + return {} + } + +export const gitlabProjectLifecycleProjector = { + id: "gitlab.project.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.project.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-project-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.project.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const action = PROJECT_LIFECYCLE_ACTIONS[sourceEvent] + if (action === undefined) + throw new Error(`GitLab projector does not recognize project event ${sourceEvent}`) + const projectData = project(signal) + const eventActor = actor(signal) + const eventServiceName = serviceName(signal) + const eventResult = result(signal) + return { + subject: projectData.path, + data: { + project: projectData, + action, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabIssueLifecycleProjector = { + id: "gitlab.issue.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.issue.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const action = ISSUE_LIFECYCLE_ACTIONS[sourceEvent as keyof typeof ISSUE_LIFECYCLE_ACTIONS] + if (action === undefined) + throw new Error(`GitLab projector does not recognize issue event ${sourceEvent}`) + const projectData = project(signal) + const issueData = issue(signal) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/issues/${issueData.iid}`, + data: { + project: projectData, + issue: { ...issueData, action }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabIssueCommentProjector = { + id: "gitlab.issue.comment", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.comment.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-comment:v1", + decodeConfig: noProjectorConfig("gitlab.issue.comment"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + if (sourceEvent !== "issue_comment") + throw new Error(`GitLab projector does not recognize issue comment event ${sourceEvent}`) + const projectData = project(signal) + const issueData = issue(signal) + const commentData = comment(signal) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/issues/${issueData.iid}#note_${commentData.id}`, + data: { + project: projectData, + issue: issueData, + comment: commentData, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabMergeRequestLifecycleProjector = { + id: "gitlab.merge-request.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.merge-request.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.merge-request.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const action = + MERGE_REQUEST_LIFECYCLE_ACTIONS[sourceEvent as keyof typeof MERGE_REQUEST_LIFECYCLE_ACTIONS] + if (action === undefined) + throw new Error(`GitLab projector does not recognize merge request event ${sourceEvent}`) + const projectData = project(signal) + const mergeRequestData = mergeRequest(signal) + if (action === "review" && mergeRequestData.reviewState === undefined) + throw new Error("GitLab projector is missing gitlab.merge_request.review_state") + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/merge_requests/${mergeRequestData.iid}`, + data: { + project: projectData, + mergeRequest: { ...mergeRequestData, action }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabMergeRequestCommentProjector = { + id: "gitlab.merge-request.comment", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.merge-request.comment.v1", + dataSchema: "urn:maple:event-schema:gitlab-merge-request-comment:v1", + decodeConfig: noProjectorConfig("gitlab.merge-request.comment"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const kind = + sourceEvent === "merge_request_comment" + ? "comment" + : sourceEvent === "merge_request_review_comment" + ? "review" + : undefined + if (kind === undefined) + throw new Error(`GitLab projector does not recognize merge request comment event ${sourceEvent}`) + const projectData = project(signal) + const mergeRequestData = mergeRequest(signal) + const commentData = comment(signal, kind) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/merge_requests/${mergeRequestData.iid}#note_${commentData.id}`, + data: { + project: projectData, + mergeRequest: mergeRequestData, + comment: commentData, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabPipelineCompletedProjector = { + id: "gitlab.pipeline.completed", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.pipeline.completed.v1", + dataSchema: "urn:maple:event-schema:gitlab-pipeline-completed:v1", + decodeConfig: noProjectorConfig("gitlab.pipeline.completed"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + if (sourceEvent !== "ci_pipeline_completed") + throw new Error(`GitLab projector does not recognize pipeline event ${sourceEvent}`) + const projectData = project(signal) + const id = positiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.id"), + "gitlab.ci.pipeline.id", + ) + const iid = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.iid"), + "gitlab.ci.pipeline.iid", + ) + const mergeRequestIid = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.merge_request_iid"), + "gitlab.ci.pipeline.merge_request_iid", + ) + const status = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.status"), + "gitlab.ci.pipeline.status", + true, + )! + if (!TERMINAL_PIPELINE_STATUSES.has(status)) + throw new Error(`GitLab projector pipeline status ${status} is not terminal`) + const name = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.name"), + "gitlab.ci.pipeline.name", + ) + const pipelineSource = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.source"), + "gitlab.ci.pipeline.source", + ) + const detailedStatus = projectorString( + field(signal, "attribute", "gitlab.ci.pipeline.detailed_status"), + "gitlab.ci.pipeline.detailed_status", + ) + const url = projectorUrl( + field(signal, "attribute", "gitlab.ci.pipeline.url"), + "gitlab.ci.pipeline.url", + true, + )! + const ref = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref") + const sha = projectorRevision( + field(signal, "attribute", "vcs.ref.head.revision"), + "vcs.ref.head.revision", + ) + const durationMs = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.duration_ms"), + "gitlab.ci.pipeline.duration_ms", + ) + const queuedDurationMs = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.queued_duration_ms"), + "gitlab.ci.pipeline.queued_duration_ms", + ) + const stageCount = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.stage_count"), + "gitlab.ci.pipeline.stage_count", + ) + const failedJobCount = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.failed_job_count"), + "gitlab.ci.pipeline.failed_job_count", + ) + const failedJobsTruncated = projectorBoolean( + field(signal, "attribute", "gitlab.ci.pipeline.failed_jobs_truncated"), + "gitlab.ci.pipeline.failed_jobs_truncated", + ) + const pipelineFailedJobs = failedJobs(signal) + if (status === "failed") { + if (failedJobCount === undefined) + throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_job_count") + if (failedJobsTruncated === undefined) + throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_jobs_truncated") + if (pipelineFailedJobs === undefined) + throw new Error("GitLab projector is missing gitlab.ci.pipeline.failed_jobs") + if (BigInt(failedJobCount) < BigInt(pipelineFailedJobs.length)) + throw new Error("GitLab projector failed job count is smaller than the summary list") + if (failedJobsTruncated !== BigInt(failedJobCount) > BigInt(pipelineFailedJobs.length)) + throw new Error("GitLab projector failed job truncation flag is inconsistent") + } else if ( + failedJobCount !== undefined || + failedJobsTruncated !== undefined || + pipelineFailedJobs !== undefined + ) { + throw new Error("GitLab projector failed job summaries are only valid for a failed pipeline") + } + const eventActor = actor(signal) + const eventServiceName = serviceName(signal) + const eventResult = result(signal) + return { + subject: `${projectData.path}/-/pipelines/${id}`, + data: { + project: projectData, + pipeline: { + id, + ...(iid === undefined ? {} : { iid }), + ...(mergeRequestIid === undefined ? {} : { mergeRequestIid }), + status, + ...(name === undefined ? {} : { name }), + ...(pipelineSource === undefined ? {} : { source: pipelineSource }), + ...(detailedStatus === undefined ? {} : { detailedStatus }), + url, + ...(ref === undefined ? {} : { ref }), + ...(sha === undefined ? {} : { sha }), + ...(durationMs === undefined ? {} : { durationMs }), + ...(queuedDurationMs === undefined ? {} : { queuedDurationMs }), + ...(stageCount === undefined ? {} : { stageCount }), + ...(failedJobCount === undefined ? {} : { failedJobCount }), + ...(failedJobsTruncated === undefined ? {} : { failedJobsTruncated }), + ...(pipelineFailedJobs === undefined ? {} : { failedJobs: pipelineFailedJobs }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabDeploymentLifecycleProjector = { + id: "gitlab.deployment.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.deployment.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.deployment.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const expectedStatus = DEPLOYMENT_STATUSES[sourceEvent as keyof typeof DEPLOYMENT_STATUSES] + if (expectedStatus === undefined) + throw new Error(`GitLab projector does not recognize deployment event ${sourceEvent}`) + const projectData = project(signal) + const id = positiveProjectorInt64( + field(signal, "attribute", "gitlab.deployment.id"), + "gitlab.deployment.id", + ) + const environment = projectorString( + field(signal, "attribute", "gitlab.deployment.environment"), + "gitlab.deployment.environment", + true, + )! + const status = projectorToken( + field(signal, "attribute", "gitlab.deployment.status"), + "gitlab.deployment.status", + true, + )! + if (status !== expectedStatus) + throw new Error(`GitLab projector deployment status ${status} conflicts with ${sourceEvent}`) + const revision = projectorRevision( + field(signal, "attribute", "gitlab.deployment.revision"), + "gitlab.deployment.revision", + true, + )! + const url = projectorUrl( + field(signal, "attribute", "gitlab.deployment.url"), + "gitlab.deployment.url", + true, + )! + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/deployments/${id}`, + data: { + project: projectData, + deployment: { + id, + environment, + status, + revision, + url, + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabJobLifecycleProjector = { + id: "gitlab.job.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.job.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-job-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.job.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const match = /^ci_job_([a-z][a-z0-9_]{0,63})$/.exec(sourceEvent) + const action = match?.[1] + if (action === undefined || !JOB_STATUSES.has(action)) + throw new Error(`GitLab projector does not recognize job event ${sourceEvent}`) + const projectData = project(signal) + const id = positiveProjectorInt64(field(signal, "attribute", "gitlab.ci.job.id"), "gitlab.ci.job.id") + const name = projectorString( + field(signal, "attribute", "gitlab.ci.job.name"), + "gitlab.ci.job.name", + true, + )! + const stage = projectorString( + field(signal, "attribute", "gitlab.ci.job.stage"), + "gitlab.ci.job.stage", + ) + const status = projectorToken( + field(signal, "attribute", "gitlab.ci.job.status"), + "gitlab.ci.job.status", + true, + )! + if (status !== action) + throw new Error(`GitLab projector job status ${status} conflicts with ${sourceEvent}`) + const url = projectorUrl(field(signal, "attribute", "gitlab.ci.job.url"), "gitlab.ci.job.url") + const pipelineId = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.ci.pipeline.id"), + "gitlab.ci.pipeline.id", + ) + const ref = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref") + const revision = projectorRevision( + field(signal, "attribute", "vcs.ref.head.revision"), + "vcs.ref.head.revision", + ) + const durationMs = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.ci.job.duration_ms"), + "gitlab.ci.job.duration_ms", + ) + const allowFailure = projectorBoolean( + field(signal, "attribute", "gitlab.ci.job.allow_failure"), + "gitlab.ci.job.allow_failure", + ) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/jobs/${id}`, + data: { + project: projectData, + job: { + id, + name, + status, + ...(stage === undefined ? {} : { stage }), + ...(url === undefined ? {} : { url }), + ...(pipelineId === undefined ? {} : { pipelineId }), + ...(ref === undefined ? {} : { ref }), + ...(revision === undefined ? {} : { revision }), + ...(durationMs === undefined ? {} : { durationMs }), + ...(allowFailure === undefined ? {} : { allowFailure }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +const REF_EVENTS = { + branch_create: { type: "branch", action: "create" }, + branch_update: { type: "branch", action: "update" }, + branch_delete: { type: "branch", action: "delete" }, + tag_create: { type: "tag", action: "create" }, + tag_update: { type: "tag", action: "update" }, + tag_delete: { type: "tag", action: "delete" }, +} as const + +export const gitlabRefLifecycleProjector = { + id: "gitlab.ref.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.ref.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-ref-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.ref.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const lifecycle = REF_EVENTS[sourceEvent as keyof typeof REF_EVENTS] + if (lifecycle === undefined) + throw new Error(`GitLab projector does not recognize ref event ${sourceEvent}`) + const projectData = project(signal) + const name = projectorString(field(signal, "attribute", "vcs.ref"), "vcs.ref", true)! + const before = projectorRevision( + field(signal, "attribute", "vcs.ref.base.revision"), + "vcs.ref.base.revision", + true, + )! + const after = projectorRevision( + field(signal, "attribute", "vcs.ref.head.revision"), + "vcs.ref.head.revision", + true, + )! + const url = projectorUrl(field(signal, "attribute", "gitlab.ref.url"), "gitlab.ref.url") + const commitCount = nonNegativeProjectorInt64( + field(signal, "attribute", "gitlab.push.commit_count"), + "gitlab.push.commit_count", + ) + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/refs/${name}`, + data: { + project: projectData, + ref: { + ...lifecycle, + name, + before, + after, + ...(url === undefined ? {} : { url }), + ...(commitCount === undefined ? {} : { commitCount }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const gitlabReleaseLifecycleProjector = { + id: "gitlab.release.lifecycle", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.release.lifecycle.v1", + dataSchema: "urn:maple:event-schema:gitlab-release-lifecycle:v1", + decodeConfig: noProjectorConfig("gitlab.release.lifecycle"), + project: (signal: NormalizedSignal) => { + const sourceEvent = eventName(signal) + const match = /^release_([a-z][a-z0-9_]{0,63})$/.exec(sourceEvent) + const action = match?.[1] + if (action === undefined || !RELEASE_ACTIONS.has(action)) + throw new Error(`GitLab projector does not recognize release event ${sourceEvent}`) + const projectData = project(signal) + const id = optionalPositiveProjectorInt64( + field(signal, "attribute", "gitlab.release.id"), + "gitlab.release.id", + ) + const tag = projectorString( + field(signal, "attribute", "gitlab.release.tag"), + "gitlab.release.tag", + true, + )! + const name = projectorString(field(signal, "attribute", "gitlab.release.name"), "gitlab.release.name") + const url = projectorUrl(field(signal, "attribute", "gitlab.release.url"), "gitlab.release.url") + const eventActor = actor(signal) + const eventResult = result(signal) + const eventServiceName = serviceName(signal) + return { + subject: `${projectData.path}/-/releases/${tag}`, + data: { + project: projectData, + release: { + ...(id === undefined ? {} : { id }), + tag, + action, + ...(name === undefined ? {} : { name }), + ...(url === undefined ? {} : { url }), + }, + sourceEvent, + ...(eventActor === undefined ? {} : { actor: eventActor }), + ...(eventResult === undefined ? {} : { result: eventResult }), + ...(eventServiceName === undefined ? {} : { serviceName: eventServiceName }), + }, + } + }, +} as const + +export const registerGitLabProjectors = (registry: ProjectorRegistry): ProjectorRegistry => + registry + .register(gitlabIssueCreatedProjector) + .register(gitlabProjectLifecycleProjector) + .register(gitlabIssueLifecycleProjector) + .register(gitlabIssueCommentProjector) + .register(gitlabMergeRequestLifecycleProjector) + .register(gitlabMergeRequestCommentProjector) + .register(gitlabPipelineCompletedProjector) + .register(gitlabDeploymentLifecycleProjector) + .register(gitlabJobLifecycleProjector) + .register(gitlabRefLifecycleProjector) + .register(gitlabReleaseLifecycleProjector) diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts new file mode 100644 index 000000000..964665a68 --- /dev/null +++ b/apps/cli/src/server/eventing/otlp.ts @@ -0,0 +1,413 @@ +import { createHash } from "node:crypto" +import { + canonicalJson, + defineSignalFields, + type JsonValue, + type NormalizedSignal, + type SignalFieldCatalogEntry, + type SignalScalar, + type SignalSourceAdapter, + type SignalSourceDefinition, +} from "@maple/eventing-core" +import { OtlpFieldError, spanIdHex, traceIdHex } from "../otlp/encode" + +interface AnyValue { + readonly stringValue?: string + readonly boolValue?: boolean + readonly intValue?: string | number + readonly doubleValue?: number + readonly bytesValue?: string + readonly arrayValue?: { readonly values?: readonly AnyValue[] } + readonly kvlistValue?: { readonly values?: readonly KeyValue[] } +} + +interface KeyValue { + readonly key?: string + readonly value?: AnyValue +} + +interface OtlpLogsRequest { + readonly resourceLogs?: readonly { + readonly resource?: { readonly attributes?: readonly KeyValue[] } + readonly scopeLogs?: readonly { + readonly scope?: { + readonly name?: string + readonly version?: string + readonly attributes?: readonly KeyValue[] + } + readonly logRecords?: readonly { + readonly timeUnixNano?: string | number + readonly observedTimeUnixNano?: string | number + readonly severityNumber?: number + readonly severityText?: string + readonly eventName?: string + readonly body?: AnyValue + readonly attributes?: readonly KeyValue[] + readonly traceId?: string + readonly spanId?: string + }[] + }[] + }[] +} + +const MAX_ATTRIBUTES = 256 +const MAX_STRING_BYTES = 16 * 1024 +const MAX_DATA_BYTES = 256 * 1024 +const MAX_VALUE_DEPTH = 8 +const MAX_VALUE_NODES = 1_024 +const SENSITIVE_KEY = + /(?:^|[._-])(authorization|cookie|password|passwd|secret|token|api[._-]?key)(?:$|[._-])/i + +const allOperators = ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"] as const +const equalityOperators = ["exists", "eq", "neq", "contains", "in"] as const + +const catalog = ( + key: string, + type: SignalScalar["type"], + operators: SignalFieldCatalogEntry["operators"] = allOperators, +): SignalFieldCatalogEntry => ({ + field: { namespace: "signal", key, type }, + operators, + sensitivity: "public", + replay: "exact", +}) + +export const OTLP_LOG_SOURCE: SignalSourceDefinition = { + sourceKind: "otel.log", + fields: [ + catalog("event.name", "string", equalityOperators), + catalog("severity.number", "int64"), + catalog("severity.text", "string", equalityOperators), + catalog("trace.id", "string", equalityOperators), + catalog("span.id", "string", equalityOperators), + catalog("time", "timestamp"), + catalog("observed_time", "timestamp"), + { + field: { namespace: "body", key: "value" }, + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + ], + openFields: [ + { + namespace: "resource", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "scope", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + ], +} + +interface ValueBudget { + nodes: number +} + +const assertStringBound = (value: string, label: string): string => { + if (Buffer.byteLength(value, "utf8") > MAX_STRING_BYTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes`) + return value +} + +const int64 = (value: string | number, label: string): string => { + if (typeof value === "number" && !Number.isSafeInteger(value)) + throw new OtlpFieldError( + `${label} must encode int64 as a decimal string when outside safe integer range`, + ) + const decimal = String(value) + if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) throw new OtlpFieldError(`${label} is not an int64`) + const parsed = BigInt(decimal) + if (parsed < -(1n << 63n) || parsed > (1n << 63n) - 1n) + throw new OtlpFieldError(`${label} is outside the int64 range`) + return decimal +} + +const anyValueScalar = (value: AnyValue | undefined, label: string): SignalScalar | null => { + if (!value) return null + if (value.stringValue !== undefined) + return { type: "string", value: assertStringBound(value.stringValue, label) } + if (value.boolValue !== undefined) return { type: "boolean", value: value.boolValue } + if (value.intValue !== undefined) return { type: "int64", value: int64(value.intValue, label) } + if (value.doubleValue !== undefined) { + if (!Number.isFinite(value.doubleValue)) throw new OtlpFieldError(`${label} must be finite`) + return { type: "float64", value: value.doubleValue } + } + return null +} + +const anyValueJson = ( + value: AnyValue | undefined, + label: string, + depth = 0, + budget: ValueBudget = { nodes: 0 }, +): JsonValue | null => { + budget.nodes += 1 + if (budget.nodes > MAX_VALUE_NODES) throw new OtlpFieldError(`${label} exceeds value node limit`) + if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError(`${label} exceeds value depth limit`) + const scalar = anyValueScalar(value, label) + if (scalar) return scalar.value + if (!value) return null + if (value.bytesValue !== undefined) return assertStringBound(value.bytesValue, `${label}.bytesValue`) + if (value.arrayValue !== undefined) + return (value.arrayValue.values ?? []).map((item, index) => + anyValueJson(item, `${label}[${index}]`, depth + 1, budget), + ) + if (value.kvlistValue !== undefined) { + const output: Record = {} + for (const [index, entry] of (value.kvlistValue.values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}.key[${index}]`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + output[key] = anyValueJson(entry.value, `${label}.${key}`, depth + 1, budget) + } + return output + } + return null +} + +interface NormalizedAttributes { + readonly scalars: ReadonlyArray<{ readonly key: string; readonly value: SignalScalar }> + readonly data: Readonly> +} + +const attributes = (values: readonly KeyValue[] | undefined, label: string): NormalizedAttributes => { + if ((values?.length ?? 0) > MAX_ATTRIBUTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_ATTRIBUTES} attributes`) + const scalars = new Map() + const data: Record = {} + for (const [index, entry] of (values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}[${index}].key`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + const scalar = anyValueScalar(entry.value, `${label}.${key}`) + if (scalar) scalars.set(key, scalar) + data[key] = anyValueJson(entry.value, `${label}.${key}`) + } + return { scalars: [...scalars].map(([key, value]) => ({ key, value })), data } +} + +const epochNanos = (value: string | number | undefined): bigint | null => { + if (value === undefined || value === "" || value === 0 || value === "0") return null + try { + const parsed = BigInt(value) + return parsed >= 0 ? parsed : null + } catch { + return null + } +} + +const nanosToTimestamp = (nanos: bigint): string => { + const seconds = nanos / 1_000_000_000n + const fraction = nanos % 1_000_000_000n + const milliseconds = Number(seconds) * 1_000 + const date = new Date(milliseconds) + if (!Number.isFinite(milliseconds) || Number.isNaN(date.getTime())) + throw new OtlpFieldError("OTLP timestamp is outside the supported date range") + return `${date.toISOString().slice(0, 19)}.${fraction.toString().padStart(9, "0")}Z` +} + +const stringAttribute = (attrs: NormalizedAttributes, key: string): string | null => { + const scalar = attrs.scalars.find((entry) => entry.key === key)?.value + return scalar?.type === "string" ? scalar.value : null +} + +const boundedIdentity = (value: string, prefix: string): string => + value.length <= 256 + ? value + : `${prefix}:sha256:${createHash("sha256").update(value, "utf8").digest("hex")}` + +const sourceUri = (resource: NormalizedAttributes, record: NormalizedAttributes): string => { + const explicit = ( + stringAttribute(record, "event.source") ?? stringAttribute(record, "cloudevents.source") + )?.trim() + if (explicit) return boundedIdentity(assertStringBound(explicit, "event source"), "urn:maple:source") + const service = stringAttribute(resource, "service.name")?.trim() + const source = service + ? `urn:maple:source:otel:${encodeURIComponent(service)}` + : "urn:maple:source:otel:local" + return boundedIdentity(source, "urn:maple:source") +} + +const sourceOccurrenceId = (record: NormalizedAttributes): string | null => { + for (const key of ["event.id", "cloudevents.id", "gitlab.event.id"]) { + const value = stringAttribute(record, key)?.trim() + if (value) return boundedIdentity(value, "source") + } + return null +} + +const derivedOccurrenceId = (input: JsonValue): string => + `derived:sha256:${createHash("sha256").update(canonicalJson(input)).digest("hex")}` + +export const normalizeOtlpLogs = ( + request: unknown, + acceptedAt = new Date().toISOString(), + tenantId = "local", +): readonly NormalizedSignal[] => { + const input = (request ?? {}) as OtlpLogsRequest + const signals: NormalizedSignal[] = [] + for (const resourceLogs of input.resourceLogs ?? []) { + const resource = attributes(resourceLogs.resource?.attributes, "resource.attributes") + for (const scopeLogs of resourceLogs.scopeLogs ?? []) { + const scope = attributes(scopeLogs.scope?.attributes, "scope.attributes") + for (const log of scopeLogs.logRecords ?? []) { + const record = attributes(log.attributes, "log.attributes") + const occurredNanos = epochNanos(log.timeUnixNano) ?? epochNanos(log.observedTimeUnixNano) + const observedNanos = epochNanos(log.observedTimeUnixNano) + const occurredAt = occurredNanos ? nanosToTimestamp(occurredNanos) : acceptedAt + const sourceObservedAt = observedNanos ? nanosToTimestamp(observedNanos) : acceptedAt + const bodyScalar = anyValueScalar(log.body, "log.body") + const traceId = traceIdHex(log.traceId, "logRecord.traceId") + const spanId = spanIdHex(log.spanId, "logRecord.spanId") + const data: JsonValue = { + resource: resource.data, + scope: { + name: assertStringBound(scopeLogs.scope?.name ?? "", "scope.name"), + version: assertStringBound(scopeLogs.scope?.version ?? "", "scope.version"), + attributes: scope.data, + }, + record: { + eventName: assertStringBound(log.eventName ?? "", "log.eventName"), + severityNumber: log.severityNumber ?? 0, + severityText: assertStringBound(log.severityText ?? "", "log.severityText"), + traceId, + spanId, + body: anyValueJson(log.body, "log.body"), + attributes: record.data, + }, + } + if (Buffer.byteLength(canonicalJson(data), "utf8") > MAX_DATA_BYTES) + throw new OtlpFieldError(`normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes`) + const source = sourceUri(resource, record) + const occurrenceId = sourceOccurrenceId(record) + const subject = + stringAttribute(record, "event.subject") ?? stringAttribute(record, "cloudevents.subject") + signals.push({ + sourceKind: "otel.log", + source, + tenantId, + occurrenceId: + occurrenceId ?? + derivedOccurrenceId({ source, occurredAt, signalKind: "otel.log", data }), + identityQuality: occurrenceId === null ? "derived" : "source", + occurredAt, + observedAt: acceptedAt, + subject, + fields: defineSignalFields([ + ...(log.eventName + ? [ + { + field: { + namespace: "signal" as const, + key: "event.name", + type: "string" as const, + }, + value: { type: "string" as const, value: log.eventName }, + }, + ] + : []), + { + field: { namespace: "signal", key: "severity.number", type: "int64" }, + value: { + type: "int64", + value: int64(log.severityNumber ?? 0, "severity.number"), + }, + }, + ...(log.severityText + ? [ + { + field: { + namespace: "signal" as const, + key: "severity.text", + type: "string" as const, + }, + value: { type: "string" as const, value: log.severityText }, + }, + ] + : []), + ...(traceId + ? [ + { + field: { + namespace: "signal" as const, + key: "trace.id", + type: "string" as const, + }, + value: { type: "string" as const, value: traceId }, + }, + ] + : []), + ...(spanId + ? [ + { + field: { + namespace: "signal" as const, + key: "span.id", + type: "string" as const, + }, + value: { type: "string" as const, value: spanId }, + }, + ] + : []), + { + field: { namespace: "signal", key: "time", type: "timestamp" }, + value: { type: "timestamp", value: occurredAt }, + }, + { + field: { namespace: "signal", key: "observed_time", type: "timestamp" }, + value: { type: "timestamp", value: sourceObservedAt }, + }, + ...resource.scalars.map(({ key, value }) => ({ + field: { namespace: "resource" as const, key, type: value.type }, + value, + })), + ...scope.scalars.map(({ key, value }) => ({ + field: { namespace: "scope" as const, key, type: value.type }, + value, + })), + ...record.scalars.map(({ key, value }) => ({ + field: { namespace: "attribute" as const, key, type: value.type }, + value, + })), + ...(bodyScalar + ? [ + { + field: { + namespace: "body" as const, + key: "value", + type: bodyScalar.type, + }, + value: bodyScalar, + }, + ] + : []), + ]), + data, + }) + } + } + } + return signals +} + +export const OTLP_LOG_ADAPTER: SignalSourceAdapter< + unknown, + { readonly acceptedAt: string; readonly tenantId: string } +> = { + definition: OTLP_LOG_SOURCE, + normalize: (raw, context) => normalizeOtlpLogs(raw, context.acceptedAt, context.tenantId), +} diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts new file mode 100644 index 000000000..0da077b85 --- /dev/null +++ b/apps/cli/src/server/eventing/runtime.ts @@ -0,0 +1,196 @@ +import { + CompiledProjectionRegistry, + ProjectorRegistry, + SignalSourceRegistry, + assertSignalProjectionInputBudget, + SignalProjectionSpecSchema, + type MapleCloudEvent, + type ProjectionFailure, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { LocalEventingControlStore } from "./control-store" +import type { EventConsumerStart } from "./control-store" +import { registerGitLabProjectors } from "./gitlab-projectors" +import { OTLP_LOG_ADAPTER } from "./otlp" +import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" + +const TENANT_ID = "local" + +export interface LocalProjectionEvaluation { + readonly events: readonly MapleCloudEvent[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +export interface LocalProjectionActivation { + readonly spec: SignalProjectionSpec + readonly next: readonly SignalProjectionSpec[] + readonly compiled: CompiledProjectionRegistry + readonly generation: number +} + +const emptyEvaluation = (): LocalProjectionEvaluation => ({ + events: [], + failures: [], + typeMismatchFields: [], +}) + +export class LocalEventingRuntime { + readonly #store: LocalEventingControlStore + readonly #sources: SignalSourceRegistry + readonly #projectors: ProjectorRegistry + readonly #telemetry: EventingTelemetry + #compiled: CompiledProjectionRegistry + #activeSourceKinds = new Set() + #generation = 0 + + constructor(store: LocalEventingControlStore, telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY) { + this.#store = store + this.#telemetry = telemetry + this.#sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) + this.#projectors = registerGitLabProjectors(new ProjectorRegistry()) + const specs = store.loadEnabledProjections(TENANT_ID) + this.#compiled = CompiledProjectionRegistry.compile(specs, this.#sources, this.#projectors) + this.#activeSourceKinds = new Set(specs.map(({ sourceKind }) => sourceKind)) + } + + hasActiveSource(sourceKind: string): boolean { + return this.#activeSourceKinds.has(sourceKind) + } + + prepareActivation(candidate: unknown): LocalProjectionActivation { + assertSignalProjectionInputBudget(candidate) + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + if (spec.tenantId !== TENANT_ID) + throw new Error(`Maple Local only accepts projections for tenant ${TENANT_ID}`) + const active = this.#store + .loadEnabledProjections(TENANT_ID) + .filter((candidate) => candidate.id !== spec.id) + const next = spec.enabled ? [...active, spec] : active + const compiled = CompiledProjectionRegistry.compile(next, this.#sources, this.#projectors) + return { spec, next, compiled, generation: this.#generation } + } + + commitActivation(activation: LocalProjectionActivation): void { + if (activation.generation !== this.#generation) + throw new Error("projection registry changed during activation; retry the request") + this.#store.saveProjection(activation.spec) + this.#compiled = activation.compiled + this.#activeSourceKinds = new Set(activation.next.map(({ sourceKind }) => sourceKind)) + this.#generation += 1 + } + + activate(candidate: unknown): void { + this.commitActivation(this.prepareActivation(candidate)) + } + + listActive(): readonly SignalProjectionSpec[] { + return this.#store.loadEnabledProjections(TENANT_ID) + } + + evaluateOtlp( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay: (rangeDate: string) => boolean = () => false, + ): LocalProjectionEvaluation { + const sourceKind = signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" + if (!this.hasActiveSource(sourceKind)) return emptyEvaluation() + const startedAt = performance.now() + const acceptedAt = new Date().toISOString() + let normalized + try { + normalized = ( + signal === "logs" + ? OTLP_LOG_ADAPTER.normalize(decoded, { acceptedAt, tenantId: TENANT_ID }) + : [] + ).filter((occurrence) => !isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) + this.#telemetry.record({ + operation: "normalization", + outcome: "success", + count: normalized.length, + durationMs: performance.now() - startedAt, + sourceKind, + }) + } catch (error) { + this.#telemetry.record({ + operation: "normalization", + outcome: "failure", + durationMs: performance.now() - startedAt, + sourceKind, + }) + throw error + } + const snapshot = this.#compiled + const events: MapleCloudEvent[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + for (const occurrence of normalized) { + const result = snapshot.evaluate(occurrence) + this.#telemetry.record({ + operation: "projection", + outcome: "success", + count: result.events.length, + sourceKind, + }) + this.#telemetry.record({ + operation: "projection", + outcome: "failure", + count: result.failures.length, + sourceKind, + }) + events.push(...result.events) + failures.push(...result.failures) + for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) + } + return { events, failures, typeMismatchFields: [...typeMismatchFields] } + } + + persistFailures(failures: readonly ProjectionFailure[]): void { + if (failures.length > 0) this.#store.recordProjectionFailures(TENANT_ID, failures) + } + + stage(events: readonly MapleCloudEvent[]) { + return this.#store.stageEvents(events) + } + + markReady(eventIds: readonly string[]): void { + this.#store.markReady(eventIds) + } + + listReady(limit?: number, after?: number) { + return this.#store.listReady(limit, after) + } + + listStaged(limit?: number, after?: number) { + return this.#store.listStaged(limit, after) + } + + listConsumers() { + return this.#store.listConsumers(TENANT_ID) + } + + registerConsumer(consumerId: string, startAt: EventConsumerStart) { + return this.#store.registerConsumer(TENANT_ID, consumerId, startAt) + } + + disableConsumer(consumerId: string) { + return this.#store.disableConsumer(TENANT_ID, consumerId) + } + + claimReady(consumerId: string, limit: number, leaseSeconds: number) { + return this.#store.claimReady(TENANT_ID, consumerId, limit, leaseSeconds) + } + + acknowledgeClaim(consumerId: string, leaseToken: string, throughSequence: number) { + return this.#store.acknowledgeClaim(TENANT_ID, consumerId, leaseToken, throughSequence) + } + + health() { + return { + activeProjections: this.listActive().length, + outboxCapacity: this.#store.outboxCapacity(), + ...this.#store.validate(), + } + } +} diff --git a/apps/cli/src/server/eventing/telemetry.ts b/apps/cli/src/server/eventing/telemetry.ts new file mode 100644 index 000000000..98ba74cbf --- /dev/null +++ b/apps/cli/src/server/eventing/telemetry.ts @@ -0,0 +1,80 @@ +import { Effect, Metric } from "effect" + +export type EventingTelemetryOperation = + | "normalization" + | "projection" + | "outbox_stage" + | "outbox_ready" + | "outbox_dedup" + | "consumer_claim" + | "consumer_ack" + | "consumer_lease" + | "consumer_lag" + +export type EventingTelemetryOutcome = + | "success" + | "failure" + | "empty" + | "active" + | "expired" + | "reclaimed" + | "observed" + +export type EventingTelemetrySourceKind = "otel.log" | "otel.span" | "otel.metric" | "unknown" + +/** Deliberately excludes tenant, consumer, event, projection, payload, and credential values. */ +export interface EventingTelemetryObservation { + readonly operation: EventingTelemetryOperation + readonly outcome: EventingTelemetryOutcome + readonly count?: number + readonly durationMs?: number + readonly lag?: number + readonly sourceKind?: EventingTelemetrySourceKind +} + +export interface EventingTelemetry { + record(observation: EventingTelemetryObservation): void +} + +export const NOOP_EVENTING_TELEMETRY: EventingTelemetry = { record: () => {} } + +const operations = Metric.counter("maple.eventing.operations_total", { + description: "Eventing operations by bounded operation and outcome", + incremental: true, +}) +const durations = Metric.histogram("maple.eventing.operation_duration_ms", { + description: "Eventing operation duration in milliseconds", + boundaries: [0.1, 0.5, 1, 5, 10, 50, 100, 500, 1_000, 5_000], +}) +const consumerLag = Metric.histogram("maple.eventing.consumer_lag_events", { + description: "Ready-event sequence lag observed by event consumers", + boundaries: [0, 1, 5, 10, 50, 100, 500, 1_000, 10_000], +}) + +export const makeEffectEventingTelemetry = ( + run: (effect: Effect.Effect) => void, +): EventingTelemetry => ({ + record(observation) { + const attributes = { + operation: observation.operation, + outcome: observation.outcome, + source_kind: observation.sourceKind ?? "unknown", + } + const effects: Effect.Effect[] = [] + const count = observation.count ?? 1 + if (Number.isFinite(count) && count > 0) + effects.push(Metric.update(Metric.withAttributes(operations, attributes), count)) + if (observation.durationMs !== undefined && Number.isFinite(observation.durationMs)) + effects.push( + Metric.update( + Metric.withAttributes(durations, attributes), + Math.max(0, observation.durationMs), + ), + ) + if (observation.lag !== undefined && Number.isSafeInteger(observation.lag)) + effects.push( + Metric.update(Metric.withAttributes(consumerLag, attributes), Math.max(0, observation.lag)), + ) + if (effects.length > 0) run(Effect.all(effects, { discard: true })) + }, +}) diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 2a2c9fee3..d2b8c17a2 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -17,6 +17,16 @@ import { rawTelemetryTtlStatements, } from "./chdb" import { buildInsertStatements } from "./inserts" +import { + eventingControlSnapshotPath, + EventConsumerConflictError, + EventConsumerInputError, + EventConsumerNotFoundError, + LocalEventingControlStore, +} from "./eventing/control-store" +import { ensureEventConsumerToken, eventConsumerTokenMatches } from "./eventing/consumer-auth" +import { LocalEventingRuntime } from "./eventing/runtime" +import { makeEffectEventingTelemetry } from "./eventing/telemetry" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { decodeLogsRequest, @@ -106,7 +116,8 @@ export const corsHeadersForAllowedOrigin = ( ? { "access-control-allow-origin": origin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": "content-type, content-encoding, authorization", + "access-control-allow-headers": + "content-type, content-encoding, authorization, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", } @@ -174,6 +185,7 @@ interface IngestResult { async function ingest( db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise { @@ -203,6 +215,17 @@ async function ingest( requestBytes, } } + let evaluation: ReturnType + try { + evaluation = eventing.evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let batches: EncodedBatch[] try { batches = encodeFor(signal, decoded) @@ -217,6 +240,18 @@ async function ingest( requestBytes, } } + let stagedEventIds: readonly string[] = [] + try { + eventing.persistFailures(evaluation.failures) + if (evaluation.events.length > 0) stagedEventIds = eventing.stage(evaluation.events).eventIds + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let rejected = 0 batches = batches.map((batch) => { const filtered = authority.filterBatch(batch.datasource, batch.ndjson) @@ -239,6 +274,15 @@ async function ingest( accepted += statement.rowCount } } + try { + if (stagedEventIds.length > 0) eventing.markReady(stagedEventIds) + } catch (error) { + return { + response: text(`event outbox readiness ${signal}: ${(error as Error).message}`, 503), + accepted, + requestBytes, + } + } const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" if (contentType.includes("json")) { const rejectedField = @@ -398,6 +442,7 @@ const ingestSpan = ( runSpan: SpanRunner, db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise => @@ -405,7 +450,7 @@ const ingestSpan = ( recoverResponse( Effect.gen(function* () { const { response, accepted, requestBytes } = yield* Effect.promise(() => - ingest(db, authority, signal, req), + ingest(db, authority, eventing, signal, req), ) yield* Effect.annotateCurrentSpan({ "http.request.body.size": requestBytes, @@ -478,7 +523,7 @@ export class RequestQuiescenceGate { } async exclusive(work: () => Promise): Promise { - if (this.#closed) throw new Error("another server maintenance operation is active") + if (this.#closed) throw new MaintenanceInProgressError() this.#closed = true try { if (this.#active > 0) await new Promise((resolve) => this.#drained.push(resolve)) @@ -489,6 +534,57 @@ export class RequestQuiescenceGate { } } +class MaintenanceInProgressError extends Error { + constructor() { + super("another server maintenance operation is active") + this.name = "MaintenanceInProgressError" + } +} + +class RequestBodyTooLargeError extends Error { + constructor(readonly maximumBytes: number) { + super(`request body exceeds ${maximumBytes} bytes`) + this.name = "RequestBodyTooLargeError" + } +} + +const readBoundedJson = async (req: Request, maximumBytes: number): Promise => { + const contentLength = req.headers.get("content-length") + if (contentLength !== null && /^[0-9]+$/.test(contentLength)) { + const declared = Number(contentLength) + if (!Number.isSafeInteger(declared) || declared > maximumBytes) + throw new RequestBodyTooLargeError(maximumBytes) + } + if (req.body === null) return JSON.parse("") as unknown + const reader = req.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > maximumBytes) { + await reader.cancel() + throw new RequestBodyTooLargeError(maximumBytes) + } + chunks.push(value) + } + } finally { + reader.releaseLock() + } + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return JSON.parse(new TextDecoder().decode(bytes)) as unknown +} + +const invalidJsonResponse = (error: unknown): Response => + error instanceof RequestBodyTooLargeError ? text(error.message, 413) : text("invalid JSON body", 400) + const admitted = async (gate: RequestQuiescenceGate, work: () => Promise): Promise => { const leave = gate.enter() if (!leave) return text("server maintenance in progress", 503) @@ -541,16 +637,26 @@ const handleRetirement = async ( } const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i +const MAX_CHECKPOINT_BODY_BYTES = 4 * 1024 +const MAX_PROJECTION_BODY_BYTES = 512 * 1024 +const MAX_CONSUMER_BODY_BYTES = 16 * 1024 /** Typed, authenticated replacement for sending BACKUP through /local/query. */ -const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Promise => { +const handleCheckpointBackup = async ( + db: Chdb, + controlStore: LocalEventingControlStore, + dataDir: string, + gate: RequestQuiescenceGate, + token: string, + req: Request, +): Promise => { if (!maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token"))) return text("maintenance authorization required", 403) let body: unknown try { - body = await req.json() - } catch { - return text("invalid JSON body", 400) + body = await readBoundedJson(req, MAX_CHECKPOINT_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) } if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) const record = body as Record @@ -558,11 +664,14 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr return text("invalid checkpoint fields", 400) if (!CHECKPOINT_ID.test(record.checkpointId)) return text("invalid checkpoint ID", 400) try { - db.exec( - `BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${record.checkpointId.toLowerCase()}/backup')`, - ) - return json({ checkpointId: record.checkpointId.toLowerCase() }) + const checkpointId = record.checkpointId.toLowerCase() + return await gate.exclusive(async () => { + const control = await controlStore.backupTo(eventingControlSnapshotPath(dataDir, checkpointId)) + db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) + return json({ checkpointId, control }) + }) } catch (error) { + if (error instanceof MaintenanceInProgressError) return text(error.message, 409) return text( `checkpoint backup failed: ${error instanceof Error ? error.message : String(error)}`, 400, @@ -570,6 +679,220 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr } } +const eventingAuthorized = (token: string, req: Request): Response | null => + maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token")) + ? null + : text("maintenance authorization required", 403) + +const handleProjectionActivation = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + token: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + let activation + try { + // Recursive schema validation and full registry compilation happen while + // normal ingest/query admission remains open. + activation = eventing.prepareActivation(body) + } catch (error) { + return text( + `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, + 400, + ) + } + try { + await gate.exclusive(async () => eventing.commitActivation(activation)) + return json({ active: eventing.listActive() }) + } catch (error) { + if (error instanceof MaintenanceInProgressError) return text(error.message, 409) + return text( + `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, + 400, + ) + } +} + +const eventConsumerErrorResponse = (error: unknown): Response => { + const message = error instanceof Error ? error.message : String(error) + if (error instanceof EventConsumerNotFoundError) return text(message, 404) + if (error instanceof EventConsumerConflictError) return text(message, 409) + if (error instanceof EventConsumerInputError) return text(message, 400) + return text(`event consumer operation failed: ${message}`, 500) +} + +const handleConsumerRegistration = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + maintenanceToken: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if ( + Object.keys(record).sort().join(",") !== "consumerId,startAt" || + typeof record.consumerId !== "string" || + (record.startAt !== "beginning" && record.startAt !== "latest") + ) + return text("invalid event consumer registration fields", 400) + const consumerId = record.consumerId as string + const startAt = record.startAt as "beginning" | "latest" + return admitted(gate, async () => { + try { + return json(eventing.registerConsumer(consumerId, startAt), 201) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerDisable = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + maintenanceToken: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if (Object.keys(record).join(",") !== "consumerId" || typeof record.consumerId !== "string") + return text("invalid event consumer disable fields", 400) + return admitted(gate, async () => { + try { + return json(eventing.disableConsumer(record.consumerId as string)) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerClaim = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + consumerToken: string, + req: Request, +): Promise => { + if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) + return text("event consumer authorization required", 403) + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if ( + Object.keys(record).sort().join(",") !== "consumerId,leaseSeconds,limit" || + typeof record.consumerId !== "string" || + typeof record.limit !== "number" || + typeof record.leaseSeconds !== "number" + ) + return text("invalid event consumer claim fields", 400) + return admitted(gate, async () => { + try { + return json( + eventing.claimReady( + record.consumerId as string, + record.limit as number, + record.leaseSeconds as number, + ), + ) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerAcknowledgement = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + consumerToken: string, + req: Request, +): Promise => { + if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) + return text("event consumer authorization required", 403) + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (typeof body !== "object" || body === null || Array.isArray(body)) return text("invalid body", 400) + const record = body as Record + if ( + Object.keys(record).sort().join(",") !== "consumerId,leaseToken,throughSequence" || + typeof record.consumerId !== "string" || + typeof record.leaseToken !== "string" || + typeof record.throughSequence !== "number" + ) + return text("invalid event consumer acknowledgement fields", 400) + return admitted(gate, async () => { + try { + return json( + eventing.acknowledgeClaim( + record.consumerId as string, + record.leaseToken as string, + record.throughSequence as number, + ), + ) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleEventingRead = ( + eventing: LocalEventingRuntime, + token: string, + req: Request, + url: URL, +): Response => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + if (url.pathname === "/local/eventing/health") return json(eventing.health()) + if (url.pathname === "/local/eventing/projections") return json(eventing.listActive()) + if (url.pathname === "/local/eventing/consumers") return json(eventing.listConsumers()) + if (url.pathname === "/local/eventing/outbox") { + const rawLimit = url.searchParams.get("limit") + const limit = rawLimit === null ? 100 : Number(rawLimit) + const rawAfter = url.searchParams.get("after") + const after = rawAfter === null ? 0 : Number(rawAfter) + const state = url.searchParams.get("state") ?? "ready" + try { + if (state === "ready") return json(eventing.listReady(limit, after)) + if (state === "staged") return json(eventing.listStaged(limit, after)) + return text("outbox state must be ready or staged", 400) + } catch (error) { + return text(error instanceof Error ? error.message : String(error), 400) + } + } + return text("not found", 404) +} + /** The `Bun.serve` fetch handler, closed over the chDB connection. Each ingest * and query request is run through `runSpan` so it leaves a trace; `/health` * and `OPTIONS` are skipped (loop-prevention convention — no health-check noise). */ @@ -581,6 +904,9 @@ const makeFetch = authority: RetiredDayAuthority, gate: RequestQuiescenceGate, maintenanceToken: string, + consumerToken: string, + controlStore: LocalEventingControlStore, + eventing: LocalEventingRuntime, ) => async (req: Request): Promise => { const url = new URL(req.url) @@ -594,18 +920,45 @@ const makeFetch = if (url.pathname === "/health") return respond(text("OK")) if (req.method === "POST") { if (url.pathname === "/v1/traces") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "traces", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "traces", req)), + ) if (url.pathname === "/v1/logs") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "logs", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "logs", req)), + ) if (url.pathname === "/v1/metrics") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "metrics", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "metrics", req)), + ) if (url.pathname === "/local/query") return respond(await admitted(gate, () => querySpan(runSpan, db, authority, req))) if (url.pathname === "/local/checkpoint/backup") - return respond(await admitted(gate, () => handleCheckpointBackup(db, maintenanceToken, req))) + return respond( + await handleCheckpointBackup( + db, + controlStore, + options.dataDir, + gate, + maintenanceToken, + req, + ), + ) + if (url.pathname === "/local/eventing/projections") + return respond(await handleProjectionActivation(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/consumers") + return respond(await handleConsumerRegistration(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/consumers/disable") + return respond(await handleConsumerDisable(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/claims") + return respond(await handleConsumerClaim(eventing, gate, consumerToken, req)) + if (url.pathname === "/local/eventing/acks") + return respond(await handleConsumerAcknowledgement(eventing, gate, consumerToken, req)) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } + if (req.method === "GET" && url.pathname.startsWith("/local/eventing/")) + return respond(handleEventingRead(eventing, maintenanceToken, req, url)) if (req.method === "GET" && options.assets) return respond(serveAsset(options.assets, url.pathname)) return respond(text("not found", 404)) } @@ -641,6 +994,32 @@ export const startServer = ( configFile: options.configFile, rawTelemetryRetentionDays: retention.effective, }) + // The request handler and synchronous eventing store share one telemetry + // runtime; eventing observations contain only bounded operation labels. + const telemetry = yield* Effect.acquireRelease( + Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), + (rt) => Effect.promise(() => rt.dispose()), + ) + const eventingTelemetry = makeEffectEventingTelemetry((effect) => { + telemetry.runFork(effect) + }) + const controlStore = yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => LocalEventingControlStore.open(options.dataDir, undefined, eventingTelemetry), + catch: (error) => + new ChdbError({ + message: `failed to open local eventing control store: ${error instanceof Error ? error.message : String(error)}`, + }), + }), + (store) => Effect.sync(() => store.close()), + ) + const eventing = yield* Effect.try({ + try: () => new LocalEventingRuntime(controlStore, eventingTelemetry), + catch: (error) => + new ChdbError({ + message: `failed to compile local event projections: ${error instanceof Error ? error.message : String(error)}`, + }), + }) // `CREATE ... IF NOT EXISTS` does not repair a table whose physical // definition was altered out of band. Inspect the opened store before the // listener is bound; a mismatch fails startup rather than allowing new @@ -699,15 +1078,14 @@ export const startServer = ( message: `failed to load maintenance token: ${error instanceof Error ? error.message : String(error)}`, }), }) + const consumerToken = yield* Effect.tryPromise({ + try: () => ensureEventConsumerToken(options.dataDir), + catch: (error) => + new ChdbError({ + message: `failed to load event consumer token: ${error instanceof Error ? error.message : String(error)}`, + }), + }) const gate = new RequestQuiescenceGate() - // A dedicated runtime carrying the OTel tracer for per-request spans: the - // Bun.serve handler runs outside Effect, so each request's span effect is - // run through this runtime. Disposed on scope close, which flushes any - // pending spans (bounded by the layer's shutdownTimeout). - const telemetry = yield* Effect.acquireRelease( - Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), - (rt) => Effect.promise(() => rt.dispose()), - ) const runSpan: SpanRunner = (effect) => telemetry.runPromise(effect) const server = yield* Effect.acquireRelease( Effect.try({ @@ -715,7 +1093,17 @@ export const startServer = ( Bun.serve({ port: options.port, hostname: options.hostname, - fetch: makeFetch(db, options, runSpan, authority, gate, maintenanceToken), + fetch: makeFetch( + db, + options, + runSpan, + authority, + gate, + maintenanceToken, + consumerToken, + controlStore, + eventing, + ), }), catch: (error) => new ServerBindError({ @@ -729,4 +1117,16 @@ export const startServer = ( return { port: server.port ?? options.port } }) -export const __testables = { recordServerResponse } +export const __testables = { + handleConsumerAcknowledgement, + handleConsumerClaim, + handleConsumerDisable, + handleConsumerRegistration, + handleCheckpointBackup, + handleEventingRead, + handleProjectionActivation, + ingest, + readBoundedJson, + recordServerResponse, + RequestQuiescenceGate, +} diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 720403aa1..946cd24bd 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,4 +1,5 @@ import { describe, it } from "@effect/vitest" +import { createHash } from "node:crypto" import { Effect, Exit, Option } from "effect" import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" import { @@ -54,6 +55,7 @@ import { import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { eventingControlSnapshotPath, LocalEventingControlStore } from "../src/server/eventing/control-store" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) @@ -306,6 +308,41 @@ describe("checkpoint IDs and strict parsers", () => { }) describe("checkpoint state resolution", () => { + it("binds a version-2 checkpoint to its eventing control snapshot", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const operationId = newCheckpointOperationId() + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "backup") + + const store = await LocalEventingControlStore.open(dataDir) + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + const controlValidation = await store.backupTo(controlPath) + store.close() + const controlBytes = readFileSync(controlPath) + writeFileSync( + join(snapshot, "manifest.json"), + `${JSON.stringify({ + ...manifest(checkpointId, operationId, dataDir), + formatVersion: 2, + backupBytes: 6, + controlRelativePath: `snapshots/${checkpointId}/control.sqlite`, + controlBytes: controlBytes.byteLength, + controlSha256: createHash("sha256").update(controlBytes).digest("hex"), + controlValidation, + })}\n`, + ) + writeState(dataDir, checkpointId) + strictEqual((await resolveCheckpoint(dataDir)).manifest.formatVersion, 2) + + const corrupted = Buffer.from(controlBytes) + corrupted[corrupted.length - 1] ^= 1 + writeFileSync(controlPath, corrupted) + await rejects(resolveCheckpoint(dataDir), /digest mismatch|quick_check failed/) + }) + }) + it("resolves immutable current, previous, and explicit IDs", async () => { await withDataDir(async (dataDir) => { const current = newCheckpointId() @@ -666,9 +703,11 @@ describe("live-store reset safety", () => { writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) mkdirSync(join(dataDir, "store"), { recursive: true }) + mkdirSync(join(dataDir, "control"), { recursive: true }) mkdirSync(join(dataDir, "metadata"), { recursive: true }) mkdirSync(join(dataDir, "tmp"), { recursive: true }) writeFileSync(join(dataDir, "store", "part.bin"), "live") + writeFileSync(join(dataDir, "control", "eventing.sqlite"), "live") writeFileSync(join(dataDir, "metadata", "table.sql"), "live") writeFileSync(join(dataDir, "status"), "live") writeFileSync(join(dataDir, "tmp", "scratch.bin"), "live") @@ -680,6 +719,7 @@ describe("live-store reset safety", () => { strictEqual((await readCheckpointState(dataDir)).current, checkpointId) ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) ok(!existsSync(join(dataDir, "store"))) + ok(!existsSync(join(dataDir, "control"))) ok(!existsSync(join(dataDir, "metadata"))) ok(!existsSync(join(dataDir, "status"))) ok(!existsSync(join(dataDir, "tmp"))) @@ -735,7 +775,7 @@ describe("live-store reset safety", () => { const checkpointId = newCheckpointId() writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) - for (const entry of ["data", "metadata", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "store", "tmp"]) { mkdirSync(join(dataDir, entry), { recursive: true }) writeFileSync(join(dataDir, entry, "live.bin"), "live") } @@ -757,7 +797,7 @@ describe("live-store reset safety", () => { ) await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) - for (const entry of ["data", "metadata", "status", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "status", "store", "tmp"]) { ok(!existsSync(join(dataDir, entry)), `${boundary}: ${entry}`) } strictEqual((await readCheckpointState(dataDir)).current, checkpointId) diff --git a/apps/cli/test/fixtures/gitlab-projector-identities.v1.json b/apps/cli/test/fixtures/gitlab-projector-identities.v1.json new file mode 100644 index 000000000..2022db7ea --- /dev/null +++ b/apps/cli/test/fixtures/gitlab-projector-identities.v1.json @@ -0,0 +1,46 @@ +[ + { + "occurrenceId": "project-event-42", + "projectionId": "gitlab-project-renamed" + }, + { + "occurrenceId": "mr-event-7", + "projectionId": "gitlab-mr-opened" + }, + { + "occurrenceId": "pipeline-event-900", + "projectionId": "gitlab-pipeline-completed" + }, + { + "occurrenceId": "delivery-issue:issue_close", + "projectionId": "gitlab-issue_close" + }, + { + "occurrenceId": "delivery-comment:0", + "projectionId": "gitlab-issue-comment" + }, + { + "occurrenceId": "delivery-mr-comment:merge_request_review_comment", + "projectionId": "gitlab-merge_request_review_comment" + }, + { + "occurrenceId": "delivery-pipeline:0", + "projectionId": "gitlab-pipeline-enriched" + }, + { + "occurrenceId": "delivery-deployment:deployment_failed", + "projectionId": "gitlab-deployment_failed" + }, + { + "occurrenceId": "delivery-job:failed", + "projectionId": "gitlab-ci_job_failed" + }, + { + "occurrenceId": "delivery-ref:tag_create", + "projectionId": "gitlab-tag_create" + }, + { + "occurrenceId": "delivery-release:create", + "projectionId": "gitlab-release_create" + } +] diff --git a/apps/cli/test/fixtures/gitlab-projectors.v1.json b/apps/cli/test/fixtures/gitlab-projectors.v1.json new file mode 100644 index 000000000..d4bf87b6c --- /dev/null +++ b/apps/cli/test/fixtures/gitlab-projectors.v1.json @@ -0,0 +1,353 @@ +[ + { + "specversion": "1.0", + "id": "sha256:fc797f0e2390dbb69749b2718831dd31f18f0412112e0d90d19fc83bf7a092ba", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.project.lifecycle.v1", + "subject": "example/widgets", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-project-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-project-renamed", + "projectionrevision": 1, + "projectorid": "gitlab.project.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "project-event-42", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets", "oldPath": "example/old-widgets" }, + "action": "renamed", + "sourceEvent": "project_rename", + "actor": { "id": "9", "name": "maintainer" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:810de97620b3c54c274b9052ef209b2a28e15d8879b88e69125c49d10aec51fe", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.merge-request.lifecycle.v1", + "subject": "example/widgets/-/merge_requests/7", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-mr-opened", + "projectionrevision": 1, + "projectorid": "gitlab.merge-request.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "mr-event-7", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "mergeRequest": { + "iid": "7", + "title": "Freeze contracts", + "url": "https://gitlab.example.test/example/widgets/-/merge_requests/7", + "action": "open", + "sourceBranch": "feature/maple", + "targetBranch": "main", + "commit": "abc123" + }, + "sourceEvent": "merge_request_open", + "actor": { "id": "9", "name": "maintainer" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:de495a092df476adba1f53296ec3f7b2966de886bf78aca7937109dc43f86ad9", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.pipeline.completed.v1", + "subject": "example/widgets/-/pipelines/900", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-pipeline-completed:v1", + "tenantid": "local", + "projectionid": "gitlab-pipeline-completed", + "projectionrevision": 1, + "projectorid": "gitlab.pipeline.completed", + "projectorversion": 1, + "sourceoccurrenceid": "pipeline-event-900", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "pipeline": { + "id": "900", + "iid": "12", + "status": "failed", + "name": "Maple CI", + "source": "push", + "detailedStatus": "failed", + "url": "https://gitlab.example.test/example/widgets/-/pipelines/900", + "ref": "main", + "sha": "deadbeef", + "durationMs": "63000", + "queuedDurationMs": "10000", + "stageCount": "3", + "failedJobCount": "0", + "failedJobsTruncated": false, + "failedJobs": [] + }, + "sourceEvent": "ci_pipeline_completed", + "actor": { "id": "9", "name": "maintainer" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:7311f08794a75eefba39bf98beb0533e8d30510525c20bf379fd1594618dad76", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.issue.lifecycle.v1", + "subject": "example/widgets/-/issues/7", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-issue-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-issue_close", + "projectionrevision": 1, + "projectorid": "gitlab.issue.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-issue:issue_close", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "issue": { + "id": "70", + "iid": "7", + "type": "issue", + "title": "Typed events", + "url": "https://gitlab.example.test/example/widgets/-/issues/7", + "labels": ["agent-ready", "backend"], + "action": "close" + }, + "sourceEvent": "issue_close", + "actor": { "id": "9", "name": "maintainer" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:d4e689dbe8e01d81500acd6ba67eb9f9dea7a251c24b8d852b2fd44573feb2f9", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.issue.comment.v1", + "subject": "example/widgets/-/issues/7#note_81", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-issue-comment:v1", + "tenantid": "local", + "projectionid": "gitlab-issue-comment", + "projectionrevision": 1, + "projectorid": "gitlab.issue.comment", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-comment:0", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "issue": { "iid": "7", "type": "issue", "labels": ["agent-ready", "backend"] }, + "comment": { + "id": "81", + "excerpt": "hello Maple", + "url": "https://gitlab.example.test/example/widgets/-/issues/7#note_81" + }, + "sourceEvent": "issue_comment", + "actor": { "id": "9", "name": "maintainer" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:b2ea1f0f683074c6fb5ce451427c873f797b2425af0fd7924117b7956dd646c3", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.merge-request.comment.v1", + "subject": "example/widgets/-/merge_requests/7#note_82", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-merge-request-comment:v1", + "tenantid": "local", + "projectionid": "gitlab-merge_request_review_comment", + "projectionrevision": 1, + "projectorid": "gitlab.merge-request.comment", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-mr-comment:merge_request_review_comment", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "mergeRequest": { + "iid": "7", + "title": "Freeze contracts", + "url": "https://gitlab.example.test/example/widgets/-/merge_requests/7" + }, + "comment": { + "id": "82", + "kind": "review", + "excerpt": "Please add a fixture", + "url": "https://gitlab.example.test/example/widgets/-/merge_requests/7#note_82" + }, + "sourceEvent": "merge_request_review_comment", + "actor": { "id": "9", "name": "maintainer" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:9ccbec35baf0a0898aa80e8b7ef29dcb294dbdbb060b895224d09f00cda2c74c", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.pipeline.completed.v1", + "subject": "example/widgets/-/pipelines/900", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-pipeline-completed:v1", + "tenantid": "local", + "projectionid": "gitlab-pipeline-enriched", + "projectionrevision": 1, + "projectorid": "gitlab.pipeline.completed", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-pipeline:0", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "pipeline": { + "id": "900", + "mergeRequestIid": "7", + "status": "failed", + "url": "https://gitlab.example.test/example/widgets/-/pipelines/900", + "failedJobCount": "3", + "failedJobsTruncated": true, + "failedJobs": [ + { + "id": "901", + "name": "unit", + "stage": "test", + "status": "failed", + "url": "https://gitlab.example.test/example/widgets/-/jobs/901" + }, + { "id": "902", "name": "integration", "stage": "test", "status": "failed" } + ] + }, + "sourceEvent": "ci_pipeline_completed", + "actor": { "id": "9", "name": "maintainer" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:7f8aea4bc391d3b954fecd9ef5356b02aaeb4289ddfa25ef9c3c16217b0d77d1", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.deployment.lifecycle.v1", + "subject": "example/widgets/-/deployments/501", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-deployment_failed", + "projectionrevision": 1, + "projectorid": "gitlab.deployment.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-deployment:deployment_failed", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "deployment": { + "id": "501", + "environment": "production", + "status": "failed", + "revision": "a8123fa", + "url": "https://gitlab.example.test/example/widgets/-/deployments/501" + }, + "sourceEvent": "deployment_failed", + "actor": { "id": "9", "name": "maintainer" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:42b44cd2784fb2e33dd5484b430cc11f429182db33907ca47e882631ddfdd18a", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.job.lifecycle.v1", + "subject": "example/widgets/-/jobs/901", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-job-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-ci_job_failed", + "projectionrevision": 1, + "projectorid": "gitlab.job.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-job:failed", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "job": { "id": "901", "name": "unit", "status": "failed" }, + "sourceEvent": "ci_job_failed", + "actor": { "id": "9", "name": "maintainer" }, + "result": "failure", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:8c3d03bcb1a4cd856324477d95f9ffbf3702aa3dc9a103c3995071eb0ac811e0", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.ref.lifecycle.v1", + "subject": "example/widgets/-/refs/refs/tags/v1.0.0", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-ref-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-tag_create", + "projectionrevision": 1, + "projectorid": "gitlab.ref.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-ref:tag_create", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "ref": { + "type": "tag", + "action": "create", + "name": "refs/tags/v1.0.0", + "before": "0000000000000000000000000000000000000000", + "after": "a8123faa8123faa8123faa8123faa8123faa8123" + }, + "sourceEvent": "tag_create", + "actor": { "id": "9", "name": "maintainer" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + }, + { + "specversion": "1.0", + "id": "sha256:9f141e93b5445145b4b7e2f88c3cfe8d53055a4d7ccf88df365415f96d43f0b5", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.release.lifecycle.v1", + "subject": "example/widgets/-/releases/v1.0.0", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-release-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-release_create", + "projectionrevision": 1, + "projectorid": "gitlab.release.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-release:create", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "release": { "tag": "v1.0.0", "action": "create", "name": "Maple 1.0" }, + "sourceEvent": "release_create", + "actor": { "id": "9", "name": "maintainer" }, + "result": "success", + "serviceName": "gitlab-repository-events" + } + } +] diff --git a/apps/cli/test/gitlab-projectors.test.ts b/apps/cli/test/gitlab-projectors.test.ts new file mode 100644 index 000000000..f3d2685b0 --- /dev/null +++ b/apps/cli/test/gitlab-projectors.test.ts @@ -0,0 +1,1044 @@ +import { deepStrictEqual, strictEqual, throws } from "node:assert" +import { describe, it } from "vitest" +import { + CompiledProjectionRegistry, + ProjectorRegistry, + SignalSourceRegistry, + makeEventId, + validateGitLabEventData, + validateMapleCloudEvent, + type SignalPredicate, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import samples from "./fixtures/gitlab-projectors.v1.json" +import identities from "./fixtures/gitlab-projector-identities.v1.json" +import { + gitlabDeploymentLifecycleProjector, + gitlabIssueCommentProjector, + gitlabIssueLifecycleProjector, + gitlabJobLifecycleProjector, + gitlabMergeRequestCommentProjector, + gitlabMergeRequestLifecycleProjector, + gitlabPipelineCompletedProjector, + gitlabProjectLifecycleProjector, + gitlabRefLifecycleProjector, + gitlabReleaseLifecycleProjector, + registerGitLabProjectors, +} from "../src/server/eventing/gitlab-projectors" +import { OTLP_LOG_ADAPTER, normalizeOtlpLogs } from "../src/server/eventing/otlp" + +const stringAttr = (key: string, value: string) => ({ key, value: { stringValue: value } }) +const intAttr = (key: string, value: string) => ({ key, value: { intValue: value } }) +const boolAttr = (key: string, value: boolean) => ({ key, value: { boolValue: value } }) +const stringArrayAttr = (key: string, values: readonly string[]) => ({ + key, + value: { arrayValue: { values: values.map((value) => ({ stringValue: value })) } }, +}) +const failedJobsAttr = ( + jobs: ReadonlyArray<{ + readonly id: string + readonly name: string + readonly stage?: string + readonly url?: string + }>, +) => ({ + key: "gitlab.ci.pipeline.failed_jobs", + value: { + arrayValue: { + values: jobs.map((job) => ({ + kvlistValue: { + values: [ + { key: "id", value: { intValue: job.id } }, + { key: "name", value: { stringValue: job.name } }, + ...(job.stage === undefined + ? [] + : [{ key: "stage", value: { stringValue: job.stage } }]), + { key: "status", value: { stringValue: "failed" } }, + ...(job.url === undefined ? [] : [{ key: "url", value: { stringValue: job.url } }]), + ], + }, + })), + }, + }, +}) + +const gitlabEvent = (eventName: string, eventId: string, extra: readonly unknown[] = []) => ({ + resourceLogs: [ + { + resource: { + attributes: [ + stringAttr("service.name", "gitlab-repository-events"), + stringAttr("service.version", "19.1.0"), + ], + }, + scopeLogs: [ + { + scope: { name: "example.gitlab.repository-events", version: "1" }, + logRecords: [ + { + timeUnixNano: "1786131720123456789", + observedTimeUnixNano: "1786131721123456789", + severityNumber: 9, + severityText: "INFO", + body: { stringValue: `gitlab event ${eventName}` }, + attributes: [ + stringAttr("event.id", eventId), + stringAttr("event.source", "https://gitlab.example.test"), + stringAttr("event.name", eventName), + stringAttr("gitlab.event.id", eventId), + stringAttr("gitlab.event.result", "success"), + intAttr("gitlab.project.id", "42"), + stringAttr("gitlab.project.path", "example/widgets"), + intAttr("gitlab.actor.id", "9"), + stringAttr("gitlab.actor.name", "maintainer"), + ...extra, + ], + }, + ], + }, + ], + }, + ], +}) + +const normalizedEvent = (request: unknown) => { + const [signal] = normalizeOtlpLogs(request, "2026-08-07T20:00:00Z") + if (!signal) throw new Error("test event did not normalize") + return signal +} + +const withoutAttribute = (request: ReturnType, key: string) => { + const copy = structuredClone(request) + copy.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.attributes = + copy.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.attributes.filter((entry) => entry.key !== key) + return copy +} + +const selectorFor = (eventName: string): SignalPredicate => ({ + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: eventName }, +}) + +const evaluate = ( + projectorId: string, + projectionId: string, + eventName: string, + signal: ReturnType, +) => { + const spec: SignalProjectionSpec = { + id: projectionId, + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: selectorFor(eventName), + projector: { id: projectorId, version: 1, config: {} }, + activeFrom: "2000-01-01T00:00:00Z", + } + const sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) + const projectors = registerGitLabProjectors(new ProjectorRegistry()) + return CompiledProjectionRegistry.compile([spec], sources, projectors).evaluate(signal) +} + +describe("GitLab event projectors", () => { + it("projects the production receiver's project lifecycle fields", () => { + const signal = normalizedEvent( + gitlabEvent("project_rename", "project-event-42", [ + stringAttr("gitlab.project.old_path", "example/old-widgets"), + ]), + ) + const result = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-renamed", + "project_rename", + signal, + ) + strictEqual(result.failures.length, 0) + strictEqual(result.events[0]?.id, samples[0]?.id) + deepStrictEqual(result.events[0]?.data, { + project: { id: "42", path: "example/widgets", oldPath: "example/old-widgets" }, + action: "renamed", + sourceEvent: "project_rename", + actor: { id: "9", name: "maintainer" }, + result: "success", + serviceName: "gitlab-repository-events", + }) + strictEqual(result.events[0]?.subject, "example/widgets") + }) + + it("maps every normalized project lifecycle event to a version-1 action", () => { + const actions = { + project_create: "created", + project_destroy: "destroyed", + project_rename: "renamed", + project_transfer: "transferred", + project_update: "updated", + project_archive: "archived", + project_unarchive: "unarchived", + project_deletion_request: "deletion_requested", + } as const + for (const [sourceEvent, action] of Object.entries(actions)) { + const result = evaluate( + "gitlab.project.lifecycle", + `gitlab-project-${sourceEvent}`, + sourceEvent, + normalizedEvent(gitlabEvent(sourceEvent, `project-${sourceEvent}`)), + ) + strictEqual(result.failures.length, 0) + const event = result.events[0] + if (!event) throw new Error(`projector did not emit ${sourceEvent}`) + strictEqual((event.data as { readonly action?: string }).action, action) + } + }) + + it("projects merge-request lifecycle data without inventing absent fields", () => { + const signal = normalizedEvent( + gitlabEvent("merge_request_open", "mr-event-7", [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), + stringAttr("gitlab.merge_request.source_branch", "feature/maple"), + stringAttr("gitlab.merge_request.target_branch", "main"), + stringAttr("gitlab.merge_request.commit", "abc123"), + ]), + ) + const result = evaluate( + "gitlab.merge-request.lifecycle", + "gitlab-mr-opened", + "merge_request_open", + signal, + ) + strictEqual(result.failures.length, 0) + strictEqual(result.events[0]?.id, samples[1]?.id) + deepStrictEqual(result.events[0]?.data, { + project: { id: "42", path: "example/widgets" }, + mergeRequest: { + iid: "7", + title: "Freeze contracts", + url: "https://gitlab.example.test/example/widgets/-/merge_requests/7", + action: "open", + sourceBranch: "feature/maple", + targetBranch: "main", + commit: "abc123", + }, + sourceEvent: "merge_request_open", + actor: { id: "9", name: "maintainer" }, + result: "success", + serviceName: "gitlab-repository-events", + }) + strictEqual(result.events[0]?.subject, "example/widgets/-/merge_requests/7") + }) + + it("projects only terminal pipeline events from the normalized receiver fields", () => { + const signal = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-event-900", [ + stringAttr("gitlab.event.result", "failure"), + intAttr("gitlab.ci.pipeline.id", "900"), + intAttr("gitlab.ci.pipeline.iid", "12"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + stringAttr("gitlab.ci.pipeline.detailed_status", "failed"), + stringAttr("gitlab.ci.pipeline.name", "Maple CI"), + stringAttr("gitlab.ci.pipeline.source", "push"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + intAttr("gitlab.ci.pipeline.failed_job_count", "0"), + boolAttr("gitlab.ci.pipeline.failed_jobs_truncated", false), + failedJobsAttr([]), + stringAttr("vcs.ref", "main"), + stringAttr("vcs.ref.head.revision", "deadbeef"), + intAttr("gitlab.ci.pipeline.duration_ms", "63000"), + intAttr("gitlab.ci.pipeline.queued_duration_ms", "10000"), + intAttr("gitlab.ci.pipeline.stage_count", "3"), + ]), + ) + const result = evaluate( + "gitlab.pipeline.completed", + "gitlab-pipeline-completed", + "ci_pipeline_completed", + signal, + ) + strictEqual(result.failures.length, 0) + strictEqual(result.events[0]?.id, samples[2]?.id) + deepStrictEqual(result.events[0]?.data, { + project: { id: "42", path: "example/widgets" }, + pipeline: { + id: "900", + iid: "12", + status: "failed", + name: "Maple CI", + source: "push", + detailedStatus: "failed", + ref: "main", + sha: "deadbeef", + durationMs: "63000", + queuedDurationMs: "10000", + stageCount: "3", + url: "https://gitlab.example.test/example/widgets/-/pipelines/900", + failedJobCount: "0", + failedJobsTruncated: false, + failedJobs: [], + }, + sourceEvent: "ci_pipeline_completed", + actor: { id: "9", name: "maintainer" }, + result: "failure", + serviceName: "gitlab-repository-events", + }) + strictEqual(result.events[0]?.subject, "example/widgets/-/pipelines/900") + }) + + it("projects every issue lifecycle action and sanitized issue comments", () => { + for (const [sourceEvent, action] of Object.entries({ + issue_open: "open", + issue_update: "update", + issue_close: "close", + issue_reopen: "reopen", + })) { + const result = evaluate( + "gitlab.issue.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-issue:${sourceEvent}`, [ + intAttr("gitlab.issue.id", "70"), + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), + stringAttr("gitlab.issue.title", "Typed events"), + stringAttr( + "gitlab.issue.url", + "https://gitlab.example.test/example/widgets/-/issues/7", + ), + stringArrayAttr("gitlab.issue.labels", ["agent-ready", "backend"]), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual((result.events[0]?.data as { issue: { action: string } }).issue.action, action) + } + + const commentResult = evaluate( + "gitlab.issue.comment", + "gitlab-issue-comment", + "issue_comment", + normalizedEvent( + gitlabEvent("issue_comment", "delivery-comment:0", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), + intAttr("gitlab.comment.id", "81"), + stringArrayAttr("gitlab.issue.labels", ["agent-ready", "backend"]), + stringAttr("gitlab.comment.excerpt", " hello\u0000\n\tMaple "), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), + ]), + ), + ) + strictEqual(commentResult.failures.length, 0) + deepStrictEqual(commentResult.events[0]?.data, { + project: { id: "42", path: "example/widgets" }, + issue: { iid: "7", type: "issue", labels: ["agent-ready", "backend"] }, + comment: { + id: "81", + excerpt: "hello Maple", + url: "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + }, + sourceEvent: "issue_comment", + actor: { id: "9", name: "maintainer" }, + result: "success", + serviceName: "gitlab-repository-events", + }) + + const workItem = evaluate( + "gitlab.issue.lifecycle", + "gitlab-work-item-updated", + "issue_update", + normalizedEvent( + gitlabEvent("issue_update", "delivery-work-item:0", [ + intAttr("gitlab.issue.iid", "19"), + stringAttr("gitlab.issue.type", "incident_response_task"), + ]), + ), + ) + deepStrictEqual((workItem.events[0]?.data as { issue: { type: string; action: string } }).issue, { + iid: "19", + type: "incident_response_task", + action: "update", + }) + }) + + it("uses an explicit MR lifecycle vocabulary and distinguishes review comments", () => { + for (const [sourceEvent, action] of Object.entries({ + merge_request_open: "open", + merge_request_update: "update", + merge_request_close: "close", + merge_request_reopen: "reopen", + merge_request_merge: "merge", + merge_request_review: "review", + })) { + const extras = [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), + ...(sourceEvent === "merge_request_review" + ? [stringAttr("gitlab.merge_request.review_state", "approved")] + : []), + ] + const result = evaluate( + "gitlab.merge-request.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent(gitlabEvent(sourceEvent, `delivery-mr:${sourceEvent}`, extras)), + ) + strictEqual(result.failures.length, 0) + strictEqual( + (result.events[0]?.data as { mergeRequest: { action: string } }).mergeRequest.action, + action, + ) + } + + for (const [sourceEvent, kind] of [ + ["merge_request_comment", "comment"], + ["merge_request_review_comment", "review"], + ] as const) { + const result = evaluate( + "gitlab.merge-request.comment", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-mr-comment:${sourceEvent}`, [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), + intAttr("gitlab.comment.id", "82"), + stringAttr("gitlab.comment.excerpt", "Please add a fixture"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7#note_82", + ), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual((result.events[0]?.data as { comment: { kind: string } }).comment.kind, kind) + } + }) + + it("projects bounded failed-job summaries with canonical pipeline metadata", () => { + const result = evaluate( + "gitlab.pipeline.completed", + "gitlab-pipeline-enriched", + "ci_pipeline_completed", + normalizedEvent( + gitlabEvent("ci_pipeline_completed", "delivery-pipeline:0", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + intAttr("gitlab.ci.pipeline.merge_request_iid", "7"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + intAttr("gitlab.ci.pipeline.failed_job_count", "3"), + boolAttr("gitlab.ci.pipeline.failed_jobs_truncated", true), + failedJobsAttr([ + { + id: "901", + name: "unit", + stage: "test", + url: "https://gitlab.example.test/example/widgets/-/jobs/901", + }, + { id: "902", name: "integration", stage: "test" }, + ]), + ]), + ), + ) + strictEqual(result.failures.length, 0) + deepStrictEqual( + (result.events[0]?.data as { pipeline: { failedJobCount: string; failedJobsTruncated: boolean } }) + .pipeline, + { + id: "900", + mergeRequestIid: "7", + status: "failed", + url: "https://gitlab.example.test/example/widgets/-/pipelines/900", + failedJobCount: "3", + failedJobsTruncated: true, + failedJobs: [ + { + id: "901", + name: "unit", + stage: "test", + status: "failed", + url: "https://gitlab.example.test/example/widgets/-/jobs/901", + }, + { id: "902", name: "integration", stage: "test", status: "failed" }, + ], + }, + ) + }) + + it("projects deployment and job lifecycle contracts with status agreement", () => { + for (const [sourceEvent, status] of Object.entries({ + deployment_running: "running", + deployment_success: "success", + deployment_failed: "failed", + deployment_canceled: "canceled", + deployment_blocked: "blocked", + deployment_manual: "manual", + })) { + const result = evaluate( + "gitlab.deployment.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-deployment:${sourceEvent}`, [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "production"), + stringAttr("gitlab.deployment.status", status), + stringAttr("gitlab.deployment.revision", "a8123fa"), + stringAttr( + "gitlab.deployment.url", + "https://gitlab.example.test/example/widgets/-/deployments/501", + ), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual( + (result.events[0]?.data as { deployment: { status: string } }).deployment.status, + status, + ) + } + + for (const status of [ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled", + ]) { + const sourceEvent = `ci_job_${status}` + const result = evaluate( + "gitlab.job.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-job:${status}`, [ + intAttr("gitlab.ci.job.id", "901"), + stringAttr("gitlab.ci.job.name", "unit"), + stringAttr("gitlab.ci.job.status", status), + ]), + ), + ) + strictEqual(result.failures.length, 0) + } + }) + + it("projects normalized branch/tag transitions and release lifecycle without descriptions", () => { + for (const [sourceEvent, type, action] of [ + ["branch_create", "branch", "create"], + ["branch_update", "branch", "update"], + ["branch_delete", "branch", "delete"], + ["tag_create", "tag", "create"], + ["tag_update", "tag", "update"], + ["tag_delete", "tag", "delete"], + ] as const) { + const result = evaluate( + "gitlab.ref.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-ref:${sourceEvent}`, [ + stringAttr("vcs.ref", type === "branch" ? "refs/heads/main" : "refs/tags/v1.0.0"), + stringAttr("vcs.ref.base.revision", "0000000000000000000000000000000000000000"), + stringAttr("vcs.ref.head.revision", "a8123faa8123faa8123faa8123faa8123faa8123"), + ]), + ), + ) + strictEqual(result.failures.length, 0) + deepStrictEqual((result.events[0]?.data as { ref: { type: string; action: string } }).ref, { + type, + action, + name: type === "branch" ? "refs/heads/main" : "refs/tags/v1.0.0", + before: "0000000000000000000000000000000000000000", + after: "a8123faa8123faa8123faa8123faa8123faa8123", + }) + } + + for (const action of ["create", "update", "delete"]) { + const sourceEvent = `release_${action}` + const result = evaluate( + "gitlab.release.lifecycle", + `gitlab-${sourceEvent}`, + sourceEvent, + normalizedEvent( + gitlabEvent(sourceEvent, `delivery-release:${action}`, [ + stringAttr("gitlab.release.tag", "v1.0.0"), + stringAttr("gitlab.release.name", "Maple 1.0"), + ]), + ), + ) + strictEqual(result.failures.length, 0) + strictEqual((result.events[0]?.data as { release: { action: string } }).release.action, action) + strictEqual("description" in (result.events[0]?.data as { release: object }).release, false) + } + }) + + it("requires source-provided identity and remains deterministic across retries for every new family", () => { + const mr = [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr("gitlab.merge_request.title", "Freeze contracts"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), + ] + const cases = [ + { + name: "project", + projector: gitlabProjectLifecycleProjector, + request: gitlabEvent("project_update", "retry-project"), + }, + { + name: "issue", + projector: gitlabIssueLifecycleProjector, + request: gitlabEvent("issue_update", "retry-issue", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "task"), + ]), + }, + { + name: "issue comment", + projector: gitlabIssueCommentProjector, + request: gitlabEvent("issue_comment", "retry-issue-comment", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "task"), + intAttr("gitlab.comment.id", "81"), + stringAttr("gitlab.comment.excerpt", "Bounded comment"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), + ]), + }, + { + name: "merge request", + projector: gitlabMergeRequestLifecycleProjector, + request: gitlabEvent("merge_request_update", "retry-mr", mr), + }, + { + name: "merge request comment", + projector: gitlabMergeRequestCommentProjector, + request: gitlabEvent("merge_request_comment", "retry-mr-comment", [ + ...mr, + intAttr("gitlab.comment.id", "82"), + stringAttr("gitlab.comment.excerpt", "Bounded review"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7#note_82", + ), + ]), + }, + { + name: "pipeline", + projector: gitlabPipelineCompletedProjector, + request: gitlabEvent("ci_pipeline_completed", "retry-pipeline", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "success"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + ]), + }, + { + name: "deployment", + projector: gitlabDeploymentLifecycleProjector, + request: gitlabEvent("deployment_success", "retry-deployment", [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "staging"), + stringAttr("gitlab.deployment.status", "success"), + stringAttr("gitlab.deployment.revision", "a8123fa"), + stringAttr( + "gitlab.deployment.url", + "https://gitlab.example.test/example/widgets/-/deployments/501", + ), + ]), + }, + { + name: "job", + projector: gitlabJobLifecycleProjector, + request: gitlabEvent("ci_job_success", "retry-job", [ + intAttr("gitlab.ci.job.id", "901"), + stringAttr("gitlab.ci.job.name", "unit"), + stringAttr("gitlab.ci.job.status", "success"), + ]), + }, + { + name: "ref", + projector: gitlabRefLifecycleProjector, + request: gitlabEvent("branch_update", "retry-ref", [ + stringAttr("vcs.ref", "refs/heads/main"), + stringAttr("vcs.ref.base.revision", "a8123fa"), + stringAttr("vcs.ref.head.revision", "b9123fa"), + ]), + }, + { + name: "release", + projector: gitlabReleaseLifecycleProjector, + request: gitlabEvent("release_update", "retry-release", [ + stringAttr("gitlab.release.tag", "v1.0.0"), + ]), + }, + ] as const + + for (const testCase of cases) { + const first = normalizedEvent(testCase.request) + const retry = normalizedEvent(testCase.request) + deepStrictEqual( + testCase.projector.project(first), + testCase.projector.project(retry), + testCase.name, + ) + throws( + () => + testCase.projector.project({ + ...first, + occurrenceId: "derived:sha256:fixture", + identityQuality: "derived", + }), + /source-provided occurrence identity/, + testCase.name, + ) + } + }) + + it("keeps source identity and the CloudEvent ID deterministic across retries", () => { + const first = normalizedEvent(gitlabEvent("project_create", "project-event-42")) + const retry = normalizedEvent(gitlabEvent("project_create", "project-event-42")) + const firstResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-created", + "project_create", + first, + ) + const retryResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-created", + "project_create", + retry, + ) + strictEqual(first.occurrenceId, "project-event-42") + strictEqual(retry.occurrenceId, "project-event-42") + strictEqual(firstResult.events[0]?.id, retryResult.events[0]?.id) + strictEqual( + firstResult.events[0]?.id, + "sha256:58a73bebd375da87aef9ce33d28376f127200a83d15d376a1d1d872d3757d653", + ) + strictEqual(firstResult.events[0]?.id?.startsWith("sha256:"), true) + }) + + it("fails closed on missing or malformed required fields", () => { + const missingProjectId = normalizedEvent( + withoutAttribute(gitlabEvent("project_create", "project-event-missing"), "gitlab.project.id"), + ) + const badProjectId = normalizedEvent( + gitlabEvent("project_create", "project-event-bad", [ + stringAttr("gitlab.project.id", "not-an-int"), + ]), + ) + const missingIdResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-missing", + "project_create", + missingProjectId, + ) + const badIdResult = evaluate( + "gitlab.project.lifecycle", + "gitlab-project-bad", + "project_create", + badProjectId, + ) + strictEqual(missingIdResult.failures.length, 1) + strictEqual(missingIdResult.failures[0]?.message, "GitLab projector is missing gitlab.project.id") + strictEqual(badIdResult.failures.length, 1) + strictEqual(badIdResult.failures[0]?.message, "GitLab projector gitlab.project.id must be an int64") + }) + + it("requires every semantically available producer handoff field", () => { + const issueWithoutType = normalizedEvent( + gitlabEvent("issue_update", "issue-no-type", [intAttr("gitlab.issue.iid", "7")]), + ) + throws(() => gitlabIssueLifecycleProjector.project(issueWithoutType), /missing gitlab.issue.type/) + + const malformedWorkItemType = normalizedEvent( + gitlabEvent("issue_update", "issue-bad-type", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "Incident Response Task"), + ]), + ) + throws(() => gitlabIssueLifecycleProjector.project(malformedWorkItemType), /lowercase token/) + + const commentWithoutExcerpt = normalizedEvent( + gitlabEvent("issue_comment", "comment-no-excerpt", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), + intAttr("gitlab.comment.id", "81"), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), + ]), + ) + throws( + () => gitlabIssueCommentProjector.project(commentWithoutExcerpt), + /missing gitlab.comment.excerpt/, + ) + + const mergeRequestWithoutTitle = normalizedEvent( + gitlabEvent("merge_request_open", "mr-no-title", [ + intAttr("gitlab.merge_request.iid", "7"), + stringAttr( + "gitlab.merge_request.url", + "https://gitlab.example.test/example/widgets/-/merge_requests/7", + ), + ]), + ) + throws( + () => gitlabMergeRequestLifecycleProjector.project(mergeRequestWithoutTitle), + /missing gitlab.merge_request.title/, + ) + + const pipelineWithoutUrl = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-no-url", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "success"), + ]), + ) + throws( + () => gitlabPipelineCompletedProjector.project(pipelineWithoutUrl), + /missing gitlab.ci.pipeline.url/, + ) + + const failedPipelineWithoutSummaryCount = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-no-summary-count", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + boolAttr("gitlab.ci.pipeline.failed_jobs_truncated", false), + failedJobsAttr([]), + ]), + ) + throws( + () => gitlabPipelineCompletedProjector.project(failedPipelineWithoutSummaryCount), + /missing gitlab.ci.pipeline.failed_job_count/, + ) + + const deploymentWithoutRevision = normalizedEvent( + gitlabEvent("deployment_success", "deployment-no-revision", [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "production"), + stringAttr("gitlab.deployment.status", "success"), + stringAttr( + "gitlab.deployment.url", + "https://gitlab.example.test/example/widgets/-/deployments/501", + ), + ]), + ) + throws( + () => gitlabDeploymentLifecycleProjector.project(deploymentWithoutRevision), + /missing gitlab.deployment.revision/, + ) + }) + + it("rejects nonterminal pipeline status, unknown actions, and unknown config fields", () => { + const running = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-running", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "running"), + ]), + ) + throws( + () => gitlabPipelineCompletedProjector.project(running), + /GitLab projector pipeline status running is not terminal/, + ) + const unknownAction = normalizedEvent( + gitlabEvent("merge_request_??", "mr-invalid", [intAttr("gitlab.merge_request.iid", "7")]), + ) + throws( + () => gitlabMergeRequestLifecycleProjector.project(unknownAction), + /GitLab projector does not recognize merge request event/, + ) + const unknownBoundedAction = normalizedEvent( + gitlabEvent("merge_request_deploy", "mr-invalid-bounded", [ + intAttr("gitlab.merge_request.iid", "7"), + ]), + ) + throws( + () => gitlabMergeRequestLifecycleProjector.project(unknownBoundedAction), + /GitLab projector does not recognize merge request event/, + ) + throws( + () => gitlabProjectLifecycleProjector.decodeConfig({ includeBody: true }), + /gitlab\.project\.lifecycle projector config contains unknown fields/, + ) + }) + + it("fails closed on inconsistent statuses, unsafe URLs, and collection bounds", () => { + const deploymentMismatch = normalizedEvent( + gitlabEvent("deployment_success", "deployment-mismatch", [ + intAttr("gitlab.deployment.id", "501"), + stringAttr("gitlab.deployment.environment", "production"), + stringAttr("gitlab.deployment.status", "failed"), + ]), + ) + throws( + () => gitlabDeploymentLifecycleProjector.project(deploymentMismatch), + /status failed conflicts with deployment_success/, + ) + + const jobMismatch = normalizedEvent( + gitlabEvent("ci_job_success", "job-mismatch", [ + intAttr("gitlab.ci.job.id", "901"), + stringAttr("gitlab.ci.job.name", "unit"), + stringAttr("gitlab.ci.job.status", "failed"), + ]), + ) + throws(() => gitlabJobLifecycleProjector.project(jobMismatch), /status failed conflicts/) + + const unsafeUrl = normalizedEvent( + gitlabEvent("issue_comment", "comment-query", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), + intAttr("gitlab.comment.id", "81"), + stringAttr("gitlab.comment.excerpt", "Safe bounded excerpt"), + stringAttr("gitlab.comment.url", "https://gitlab.example.test/note?private=value"), + ]), + ) + throws(() => gitlabIssueCommentProjector.project(unsafeUrl), /must not contain a query or fragment/) + + const tooManyJobs = normalizedEvent( + gitlabEvent("ci_pipeline_completed", "pipeline-too-many-jobs", [ + intAttr("gitlab.ci.pipeline.id", "900"), + stringAttr("gitlab.ci.pipeline.status", "failed"), + stringAttr( + "gitlab.ci.pipeline.url", + "https://gitlab.example.test/example/widgets/-/pipelines/900", + ), + failedJobsAttr( + Array.from({ length: 21 }, (_, index) => ({ + id: String(index + 1), + name: `job-${index}`, + })), + ), + ]), + ) + throws(() => gitlabPipelineCompletedProjector.project(tooManyJobs), /exceeds 20 jobs/) + + const oversizedExcerpt = normalizedEvent( + gitlabEvent("issue_comment", "comment-too-long", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), + intAttr("gitlab.comment.id", "81"), + stringAttr("gitlab.comment.excerpt", "x".repeat(1025)), + stringAttr( + "gitlab.comment.url", + "https://gitlab.example.test/example/widgets/-/issues/7#note_81", + ), + ]), + ) + throws(() => gitlabIssueCommentProjector.project(oversizedExcerpt), /exceeds 1024 UTF-8 bytes/) + + const tooManyLabels = normalizedEvent( + gitlabEvent("issue_open", "issue-too-many-labels", [ + intAttr("gitlab.issue.iid", "7"), + stringAttr("gitlab.issue.type", "issue"), + stringArrayAttr( + "gitlab.issue.labels", + Array.from({ length: 51 }, (_, index) => `label-${index}`), + ), + ]), + ) + const labelsResult = evaluate( + "gitlab.issue.lifecycle", + "gitlab-issue-labels-bounded", + "issue_open", + tooManyLabels, + ) + strictEqual( + labelsResult.failures[0]?.message, + "GitLab projector gitlab.issue.labels exceeds 50 labels", + ) + }) + + it("preserves indexed producer occurrence IDs as distinct deterministic CloudEvent identities", () => { + const request = gitlabEvent("project_update", "delivery-uuid:0") + const second = structuredClone(request.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!) + second.attributes = second.attributes.map((entry) => + entry.key === "event.id" || entry.key === "gitlab.event.id" + ? stringAttr(entry.key, "delivery-uuid:1") + : entry, + ) + request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(second) + const signals = normalizeOtlpLogs(request, "2026-08-07T20:00:00Z") + deepStrictEqual( + signals.map(({ occurrenceId }) => occurrenceId), + ["delivery-uuid:0", "delivery-uuid:1"], + ) + const ids = signals.map( + (signal) => + evaluate("gitlab.project.lifecycle", "gitlab-project-updated", "project_update", signal) + .events[0]?.id, + ) + strictEqual(new Set(ids).size, 2) + deepStrictEqual( + ids, + signals.map( + (signal) => + evaluate("gitlab.project.lifecycle", "gitlab-project-updated", "project_update", signal) + .events[0]?.id, + ), + ) + }) + + it("keeps the checked-in sample CloudEvents envelope-valid", () => { + strictEqual(samples.length, 11) + strictEqual(identities.length, samples.length) + for (const [index, sample] of samples.entries()) { + strictEqual(validateMapleCloudEvent(sample).event.id, sample.id) + validateGitLabEventData(sample.dataschema, sample.data) + const identity = identities[index]! + strictEqual( + makeEventId({ + tenantId: "local", + sourceKind: "otel.log", + source: "https://gitlab.example.test", + occurrenceId: identity.occurrenceId, + projectionId: identity.projectionId, + projectionRevision: 1, + }), + sample.id, + ) + } + }) +}) diff --git a/apps/cli/test/local-eventing-consumer-auth.test.ts b/apps/cli/test/local-eventing-consumer-auth.test.ts new file mode 100644 index 000000000..6e264bdcd --- /dev/null +++ b/apps/cli/test/local-eventing-consumer-auth.test.ts @@ -0,0 +1,48 @@ +import { strictEqual } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import { + ensureEventConsumerToken, + eventConsumerTokenMatches, + eventConsumerTokenPath, +} from "../src/server/eventing/consumer-auth" + +describe("local event consumer authorization", () => { + it("creates a stable private token separate from the data directory", async () => { + const parent = mkdtempSync(join(tmpdir(), "maple-event-consumer-auth-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir) + try { + const first = await ensureEventConsumerToken(dataDir) + const second = await ensureEventConsumerToken(dataDir) + strictEqual(first.length, 64) + strictEqual(second, first) + strictEqual(statSync(eventConsumerTokenPath(dataDir)).mode & 0o777, 0o600) + strictEqual(eventConsumerTokenMatches(first, first), true) + strictEqual(eventConsumerTokenMatches(first, `${first}0`), false) + strictEqual(eventConsumerTokenMatches(first, null), false) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) + + it("refuses a symlink in place of the token", async () => { + const parent = mkdtempSync(join(tmpdir(), "maple-event-consumer-auth-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir) + try { + symlinkSync(join(parent, "target"), eventConsumerTokenPath(dataDir)) + let message = "" + try { + await ensureEventConsumerToken(dataDir) + } catch (error) { + message = error instanceof Error ? error.message : String(error) + } + strictEqual(message.includes("not a real file"), true) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts new file mode 100644 index 000000000..c6c10ebda --- /dev/null +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -0,0 +1,540 @@ +import { deepStrictEqual, ok, rejects, strictEqual, throws } from "node:assert" +import { Database } from "bun:sqlite" +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import type { MapleCloudEvent, SignalProjectionSpec } from "@maple/eventing-core" +import { eventingControlPath, LocalEventingControlStore } from "../src/server/eventing/control-store" +import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-control-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 1, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + projector: { id: "gitlab.issue", version: 1, config: { includeTitle: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const event = (overrides: Partial = {}): MapleCloudEvent => ({ + specversion: "1.0", + id: "sha256:061c0b5d99b92ef65ab8813c6d84988e4b1582e705e0077c952e62a0e84b6b08", + source: "urn:maple:source:otel:local", + type: "dev.maple.gitlab.issue.created.v1", + subject: "project/example/issues/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:gitlab-issue:v1", + tenantid: "tenant-a", + projectionid: "gitlab-issue-created", + projectionrevision: 1, + projectorid: "gitlab.issue", + projectorversion: 1, + data: { iid: 42, title: "Example" }, + ...overrides, +}) + +describe("LocalEventingControlStore", () => { + it("records bounded outbox and consumer telemetry without identifiers or payloads", async () => + withDataDir(async (dataDir) => { + const observations: EventingTelemetryObservation[] = [] + const store = await LocalEventingControlStore.open(dataDir, undefined, { + record: (observation) => observations.push(observation), + }) + try { + const sensitiveEvent = event({ + data: { iid: 42, title: "PAYLOAD-MUST-NOT-BE-METRIC-DATA" }, + }) + store.stageEvents([sensitiveEvent, sensitiveEvent]) + throws(() => store.stageEvents([event({ data: { iid: 43 } })]), /collision/) + throws(() => store.markReady(["unknown-event-identifier"]), /unknown event/) + store.markReady([sensitiveEvent.id]) + store.registerConsumer("tenant-a", "private-consumer-identifier", "beginning") + const claim = store.claimReady("tenant-a", "private-consumer-identifier", 10, 30) + throws( + () => store.claimReady("tenant-a", "private-consumer-identifier", 10, 30), + /active lease/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + "incorrect-private-token", + claim.throughSequence!, + ), + /token does not match/, + ) + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + claim.leaseToken!, + claim.throughSequence!, + ) + + const operationOutcomes = observations.map( + ({ operation, outcome }) => `${operation}:${outcome}`, + ) + for (const expected of [ + "outbox_stage:success", + "outbox_stage:failure", + "outbox_ready:success", + "outbox_ready:failure", + "outbox_dedup:success", + "consumer_claim:success", + "consumer_claim:failure", + "consumer_ack:success", + "consumer_ack:failure", + "consumer_lease:failure", + "consumer_lag:observed", + ]) + ok(operationOutcomes.includes(expected), `missing telemetry observation ${expected}`) + + const serialized = JSON.stringify(observations) + for (const forbidden of [ + "PAYLOAD-MUST-NOT-BE-METRIC-DATA", + "private-consumer-identifier", + "incorrect-private-token", + sensitiveEvent.id, + claim.leaseToken!, + ]) + strictEqual(serialized.includes(forbidden), false) + } finally { + store.close() + } + })) + + it("stores immutable sequential revisions and only loads the active revision", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + throws( + () => + store.saveProjection( + projection({ projector: { id: "changed", version: 1, config: {} } }), + ), + /immutable/, + ) + throws(() => store.saveProjection(projection({ revision: 3 })), /must be 2/) + + store.saveProjection(projection({ revision: 2, enabled: false })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), []) + store.saveProjection(projection({ revision: 3 })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) + deepStrictEqual(store.validate(), { + schemaVersion: 2, + projectionRevisions: 3, + projectionFailures: 0, + stagedEvents: 0, + readyEvents: 0, + }) + } finally { + store.close() + } + })) + + it("deduplicates staged events, rejects collisions, and preserves ready order", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + deepStrictEqual(store.stageEvents([event(), event()]), { + inserted: 1, + deduplicated: 1, + eventIds: [event().id, event().id], + }) + throws(() => store.stageEvents([event({ data: { iid: 43 } })]), /collision/) + throws(() => store.markReady(["unknown"]), /unknown event/) + store.markReady([event().id]) + store.markReady([event().id]) + deepStrictEqual(store.listStaged().events, []) + deepStrictEqual( + store.listReady().events.map(({ event }) => event), + [event()], + ) + } finally { + store.close() + } + })) + + it("survives restart and round-trips through a validated standalone snapshot", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.saveProjection(projection()) + store.stageEvents([event()]) + store.markReady([event().id]) + store.recordProjectionFailures("tenant-a", [ + { + projectionId: "gitlab-issue-created", + projectionRevision: 1, + occurrenceId: "issue-42", + message: "test failure", + }, + ]) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual( + store.listReady().events.map(({ event }) => event), + [event()], + ) + const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") + const validation = await store.backupTo(snapshot) + deepStrictEqual(validation, { + schemaVersion: 2, + projectionRevisions: 1, + projectionFailures: 1, + stagedEvents: 0, + readyEvents: 1, + }) + store.close() + + const restored = join(dataDir, "restored") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + deepStrictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(restored)), + validation, + ) + const restoredStore = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual( + restoredStore.listReady().events.map(({ event }) => event), + [event()], + ) + } finally { + restoredStore.close() + } + })) + + it("checkpoints committed live WAL state before serializing", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + store.stageEvents([event()]) + store.markReady([event().id]) + const walPath = `${eventingControlPath(dataDir)}-wal` + ok(existsSync(walPath)) + ok(statSync(walPath).size > 0, "test requires uncheckpointed WAL frames") + + const snapshot = join(dataDir, "backups", "live-wal", "control.sqlite") + await store.backupTo(snapshot) + strictEqual(statSync(walPath).size, 0) + + const restored = join(dataDir, "restored-live-wal") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + const restoredStore = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual( + restoredStore.listReady().events.map(({ event }) => event), + [event()], + ) + } finally { + restoredStore.close() + } + } finally { + store.close() + } + })) + + it("paginates every ready event and applies fail-closed outbox capacity", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 2, + maxOutboxBytes: 1024 * 1024, + }) + try { + const second = event({ id: "event-2", data: { iid: 43, title: "Second" } }) + const third = event({ id: "event-3", data: { iid: 44, title: "Third" } }) + const staged = store.stageEvents([event(), second]) + store.markReady(staged.eventIds) + + const firstPage = store.listReady(1) + strictEqual(firstPage.events.length, 1) + strictEqual(firstPage.nextCursor, firstPage.events[0]?.sequence) + const secondPage = store.listReady(1, firstPage.nextCursor!) + deepStrictEqual( + [...firstPage.events, ...secondPage.events].map(({ event }) => event.id), + [event().id, second.id], + ) + strictEqual(secondPage.nextCursor, null) + deepStrictEqual(store.stageEvents([event()]).deduplicated, 1) + throws(() => store.stageEvents([third]), /outbox capacity exceeded/) + } finally { + store.close() + } + })) + + it("pages recovered events by first readiness transition instead of staging order", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const first = event({ id: "event-a" }) + const second = event({ id: "event-b" }) + store.stageEvents([first]) + store.stageEvents([second]) + store.markReady([second.id]) + + const initialPage = store.listReady(1) + deepStrictEqual( + initialPage.events.map(({ event }) => event.id), + [second.id], + ) + const cursor = initialPage.events[0]!.sequence + + store.markReady([first.id]) + const recoveredPage = store.listReady(1, cursor) + deepStrictEqual( + recoveredPage.events.map(({ event }) => event.id), + [first.id], + ) + strictEqual(recoveredPage.events[0]!.sequence > cursor, true) + } finally { + store.close() + } + })) + + it("migrates schema 1 in place and keeps schema-1 snapshots restorable", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.stageEvents([event()]) + store.markReady([event().id]) + store.close() + + const database = new Database(eventingControlPath(dataDir), { + readwrite: true, + strict: true, + safeIntegers: true, + }) + database.exec("DROP TABLE event_consumers") + database.exec("PRAGMA user_version = 1") + database.close(true) + + strictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(dataDir)).schemaVersion, + 1, + ) + store = await LocalEventingControlStore.open(dataDir) + try { + strictEqual(store.validate().schemaVersion, 2) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [event().id], + ) + deepStrictEqual(store.listConsumers("tenant-a"), []) + } finally { + store.close() + } + })) + + it("leases whole batches, redelivers after expiry, and rejects stale acknowledgements", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 10, + maxOutboxBytes: 1024 * 1024, + retainAcknowledgedReadyEvents: 0, + }) + try { + const second = event({ id: "event-2" }) + const third = event({ id: "event-3" }) + const staged = store.stageEvents([event(), second, third]) + store.markReady(staged.eventIds) + store.registerConsumer("tenant-a", "matrix", "beginning", "2026-08-13T12:00:00.000Z") + + const firstClaim = store.claimReady("tenant-a", "matrix", 2, 10, "2026-08-13T12:00:01.000Z") + strictEqual(firstClaim.leaseToken?.length, 64) + deepStrictEqual( + firstClaim.events.map(({ event }) => event.id), + [event().id, second.id], + ) + throws( + () => store.claimReady("tenant-a", "matrix", 2, 10, "2026-08-13T12:00:02.000Z"), + /active lease/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "matrix", + "0".repeat(64), + firstClaim.throughSequence!, + "2026-08-13T12:00:03.000Z", + ), + /token does not match/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "matrix", + firstClaim.leaseToken!, + firstClaim.events[0]!.sequence, + "2026-08-13T12:00:03.000Z", + ), + /complete claimed batch/, + ) + + const retry = store.claimReady("tenant-a", "matrix", 2, 10, "2026-08-13T12:00:12.000Z") + deepStrictEqual( + retry.events.map(({ event }) => event.id), + [event().id, second.id], + ) + strictEqual(retry.leaseToken === firstClaim.leaseToken, false) + deepStrictEqual( + store.acknowledgeClaim( + "tenant-a", + "matrix", + retry.leaseToken!, + retry.throughSequence!, + "2026-08-13T12:00:13.000Z", + ), + { + consumerId: "matrix", + acknowledgedThrough: retry.throughSequence, + prunedEvents: 2, + }, + ) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [third.id], + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "matrix", + firstClaim.leaseToken!, + firstClaim.throughSequence!, + "2026-08-13T12:00:14.000Z", + ), + /no active lease/, + ) + } finally { + store.close() + } + })) + + it("prunes only after every active consumer advances and never prunes staged events", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 10, + maxOutboxBytes: 1024 * 1024, + retainAcknowledgedReadyEvents: 0, + }) + try { + const second = event({ id: "event-2" }) + const third = event({ id: "event-3" }) + const stranded = event({ id: "event-staged" }) + const ready = store.stageEvents([event(), second, third]) + store.markReady(ready.eventIds) + store.stageEvents([stranded]) + store.registerConsumer("tenant-a", "matrix-a", "beginning") + store.registerConsumer("tenant-a", "matrix-b", "beginning") + + const fast = store.claimReady("tenant-a", "matrix-a", 3, 30) + strictEqual( + store.acknowledgeClaim("tenant-a", "matrix-a", fast.leaseToken!, fast.throughSequence!) + .prunedEvents, + 0, + ) + const slow = store.claimReady("tenant-a", "matrix-b", 2, 30) + strictEqual( + store.acknowledgeClaim("tenant-a", "matrix-b", slow.leaseToken!, slow.throughSequence!) + .prunedEvents, + 2, + ) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [third.id], + ) + store.disableConsumer("tenant-a", "matrix-b") + deepStrictEqual(store.listReady().events, []) + deepStrictEqual( + store.listStaged().events.map(({ event }) => event.id), + [stranded.id], + ) + } finally { + store.close() + } + })) + + it("starts latest consumers after backlog and checkpoints active leases", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.stageEvents([event()]) + store.markReady([event().id]) + const registered = store.registerConsumer( + "tenant-a", + "matrix", + "latest", + "2099-01-01T00:00:00.000Z", + ) + strictEqual(registered.lastAcknowledgedSequence, store.listReady().events[0]!.sequence) + deepStrictEqual( + store.claimReady("tenant-a", "matrix", 10, 300, "2099-01-01T00:00:01.000Z").events, + [], + ) + + const second = event({ id: "event-2" }) + store.stageEvents([second]) + store.markReady([second.id]) + const claim = store.claimReady("tenant-a", "matrix", 10, 300, "2099-01-01T00:00:02.000Z") + const snapshot = join(dataDir, "backups", "consumer", "control.sqlite") + await store.backupTo(snapshot) + store.close() + + const restored = join(dataDir, "restored-consumer") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + store = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual( + store.listConsumers("tenant-a")[0]?.claimedThroughSequence, + claim.throughSequence, + ) + strictEqual( + store.acknowledgeClaim( + "tenant-a", + "matrix", + claim.leaseToken!, + claim.throughSequence!, + "2099-01-01T00:00:03.000Z", + ).acknowledgedThrough, + claim.throughSequence, + ) + } finally { + store.close() + } + })) + + it("refuses a symlink in place of the database", async () => + withDataDir(async (dataDir) => { + const controlPath = eventingControlPath(dataDir) + mkdirSync(join(dataDir, "control"), { recursive: true }) + symlinkSync(join(dataDir, "target.sqlite"), controlPath) + await rejects(() => LocalEventingControlStore.open(dataDir), /not a real file/) + strictEqual(controlPath.endsWith("control/eventing.sqlite"), true) + })) +}) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts new file mode 100644 index 000000000..46e486f10 --- /dev/null +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -0,0 +1,374 @@ +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" +import { describe, it } from "vitest" +import { __testables } from "../src/server/serve" + +describe("Local eventing ingest seam", () => { + it("requires maintenance authorization and exposes staged records only when requested", async () => { + const eventing = { + health: () => ({ activeProjections: 1 }), + listActive: () => [], + listReady: () => ({ events: [{ sequence: 1, event: { id: "ready" } }], nextCursor: null }), + listStaged: (_limit: number, after: number) => ({ + events: [{ sequence: after + 1, event: { id: "staged" } }], + nextCursor: null, + }), + } + const unauthorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), + new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + ) + strictEqual(unauthorized.status, 403) + + const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged&after=41", { + headers: { "x-maple-maintenance-token": "maintenance-secret" }, + }) + const authorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + request, + new URL(request.url), + ) + strictEqual(authorized.status, 200) + deepStrictEqual(await authorized.json(), { + events: [{ sequence: 42, event: { id: "staged" } }], + nextCursor: null, + }) + }) + + it("authenticates and reads activation bodies before closing admission", async () => { + const gate = new __testables.RequestQuiescenceGate() + const neverClosed = new ReadableStream() + const unauthorized = await __testables.handleProjectionActivation( + {} as never, + gate, + "maintenance-secret", + { + headers: new Headers(), + body: neverClosed, + } as Request, + ) + strictEqual(unauthorized.status, 403) + const afterUnauthorized = gate.enter() + ok(afterUnauthorized, "invalid authorization must not close admission") + afterUnauthorized() + + const checkpointUnauthorized = await __testables.handleCheckpointBackup( + {} as never, + {} as never, + "/unused", + gate, + "maintenance-secret", + { headers: new Headers(), body: neverClosed } as Request, + ) + strictEqual(checkpointUnauthorized.status, 403) + const afterCheckpointUnauthorized = gate.enter() + ok(afterCheckpointUnauthorized, "checkpoint authorization must precede exclusivity") + afterCheckpointUnauthorized() + + let controller!: ReadableStreamDefaultController + const slowBody = new ReadableStream({ + start(value) { + controller = value + }, + }) + let committed = false + const pending = __testables.handleProjectionActivation( + { + prepareActivation: (body: unknown) => ({ body }), + commitActivation: () => { + committed = true + }, + listActive: () => [], + } as never, + gate, + "maintenance-secret", + { + headers: new Headers({ "x-maple-maintenance-token": "maintenance-secret" }), + body: slowBody, + } as Request, + ) + await Promise.resolve() + const whileReading = gate.enter() + ok(whileReading, "an incomplete request body must not close admission") + whileReading() + controller.enqueue(new TextEncoder().encode("{}")) + controller.close() + strictEqual((await pending).status, 200) + strictEqual(committed, true) + }) + + it("bounds activation bodies and reports concurrent maintenance intentionally", async () => { + const oversized = new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + body: "123456789", + }) + await rejects(() => __testables.readBoundedJson(oversized, 8), /exceeds 8 bytes/) + + const gate = new __testables.RequestQuiescenceGate() + let releaseMaintenance!: () => void + const maintenance = gate.exclusive( + () => + new Promise((resolve) => { + releaseMaintenance = resolve + }), + ) + await Promise.resolve() + const response = await __testables.handleProjectionActivation( + { + prepareActivation: () => ({}), + commitActivation: () => undefined, + listActive: () => [], + } as never, + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: "{}", + }), + ) + strictEqual(response.status, 409) + releaseMaintenance() + await maintenance + }) + + it("separates consumer administration from claim and acknowledgement authorization", async () => { + const gate = new __testables.RequestQuiescenceGate() + const calls: string[] = [] + const eventing = { + registerConsumer: (consumerId: string, startAt: string) => { + calls.push(`register:${consumerId}:${startAt}`) + return { consumerId, active: true } + }, + disableConsumer: (consumerId: string) => { + calls.push(`disable:${consumerId}`) + return { consumerId, active: false } + }, + claimReady: (consumerId: string, limit: number, leaseSeconds: number) => { + calls.push(`claim:${consumerId}:${limit}:${leaseSeconds}`) + return { + consumerId, + leaseToken: "a".repeat(64), + throughSequence: 7, + events: [{ sequence: 7, event: { id: "event-7" } }], + } + }, + acknowledgeClaim: (consumerId: string, _leaseToken: string, throughSequence: number) => { + calls.push(`ack:${consumerId}:${throughSequence}`) + return { consumerId, acknowledgedThrough: throughSequence, prunedEvents: 0 } + }, + } + + const registration = await __testables.handleConsumerRegistration( + eventing as never, + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/consumers", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "matrix", startAt: "beginning" }), + }), + ) + strictEqual(registration.status, 201) + + const wrongClaimCredential = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "matrix", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(wrongClaimCredential.status, 403) + + const claim = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "matrix", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(claim.status, 200) + const claimed = (await claim.json()) as { leaseToken: string; throughSequence: number } + + const acknowledgement = await __testables.handleConsumerAcknowledgement( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/acks", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ + consumerId: "matrix", + leaseToken: claimed.leaseToken, + throughSequence: claimed.throughSequence, + }), + }), + ) + strictEqual(acknowledgement.status, 200) + deepStrictEqual(calls, ["register:matrix:beginning", "claim:matrix:10:30", "ack:matrix:7"]) + + let releaseMaintenance!: () => void + const maintenance = gate.exclusive( + () => + new Promise((resolve) => { + releaseMaintenance = resolve + }), + ) + await Promise.resolve() + const blockedClaim = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "matrix", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(blockedClaim.status, 503) + releaseMaintenance() + await maintenance + }) + + it("isolates projection failures, stores telemetry, and makes sibling events ready", async () => { + const order: string[] = [] + const event = { id: "event-1" } + const db = { + exec: () => { + order.push("chdb-insert") + }, + } + const authority = { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => { + order.push("retention-filter") + return { ndjson, accepted: 1, rejected: 0 } + }, + } + const eventing = { + evaluateOtlp: () => { + order.push("evaluate") + return { + events: [event], + failures: [ + { + projectionId: "oversized-projector", + projectionRevision: 1, + occurrenceId: "occurrence-1", + message: "CloudEvent exceeds 262144 UTF-8 bytes", + }, + ], + typeMismatchFields: [], + } + }, + persistFailures: () => order.push("persist-failures"), + stage: () => { + order.push("stage") + return { inserted: 1, deduplicated: 0, eventIds: [event.id] } + }, + markReady: () => order.push("ready"), + } + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786131720123456789", + body: { stringValue: "one" }, + }, + ], + }, + ], + }, + ], + }), + }) + + const result = await __testables.ingest( + db as never, + authority as never, + eventing as never, + "logs", + request, + ) + strictEqual(result.response.status, 200) + strictEqual(result.accepted, 1) + deepStrictEqual(order, [ + "evaluate", + "persist-failures", + "stage", + "retention-filter", + "chdb-insert", + "ready", + ]) + }) + + it("leaves a staged event non-ready when the warehouse write fails", async () => { + let markedReady = false + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "one" } }] }] }], + }), + }) + const result = await __testables.ingest( + { + exec: () => { + throw new Error("write failed") + }, + } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + { + evaluateOtlp: () => ({ events: [{ id: "event-1" }], failures: [], typeMismatchFields: [] }), + persistFailures: () => undefined, + stage: () => ({ inserted: 1, deduplicated: 0, eventIds: ["event-1"] }), + markReady: () => { + markedReady = true + }, + } as never, + "logs", + request, + ) + strictEqual(result.response.status, 500) + strictEqual(markedReady, false) + }) +}) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts new file mode 100644 index 000000000..f9ea5300b --- /dev/null +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -0,0 +1,311 @@ +import { deepStrictEqual, ok, strictEqual, throws } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import type { SignalProjectionSpec } from "@maple/eventing-core" +import { LocalEventingControlStore } from "../src/server/eventing/control-store" +import { normalizeOtlpLogs } from "../src/server/eventing/otlp" +import { LocalEventingRuntime } from "../src/server/eventing/runtime" +import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" +import { encodeLogs } from "../src/server/otlp/encode" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-runtime-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const attr = (key: string, value: Record) => ({ key, value }) + +const gitlabIssueCreated = { + resourceLogs: [ + { + resource: { + attributes: [ + attr("service.name", { stringValue: "gitlab-rails" }), + attr("service.version", { stringValue: "19.1.0" }), + ], + }, + scopeLogs: [ + { + scope: { name: "gitlab.event_store", version: "1.0.0" }, + logRecords: [ + { + timeUnixNano: "1786131720123456789", + observedTimeUnixNano: "1786131721123456789", + eventName: "gitlab.issue.created", + severityNumber: 9, + severityText: "INFO", + body: { stringValue: "Issue 42 created" }, + attributes: [ + attr("event.id", { stringValue: "01K20GITLABISSUE42" }), + attr("event.source", { stringValue: "https://gitlab.example.test" }), + attr("gitlab.project.id", { intValue: "7" }), + attr("gitlab.project.path", { stringValue: "example/widgets" }), + attr("gitlab.issue.id", { intValue: "4200" }), + attr("gitlab.issue.iid", { intValue: "42" }), + attr("gitlab.issue.title", { stringValue: "Wire GitLab events" }), + attr("gitlab.issue.url", { + stringValue: "https://gitlab.example.test/example/widgets/-/issues/42", + }), + attr("gitlab.user.id", { intValue: "9" }), + attr("gitlab.user.username", { stringValue: "operator" }), + ], + }, + ], + }, + ], + }, + ], +} + +const firstLogRecord = (request: typeof gitlabIssueCreated) => + request.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]! + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: { + op: "all", + clauses: [ + { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + { + op: "gte", + field: { namespace: "attribute", key: "gitlab.issue.iid", type: "int64" }, + value: { type: "int64", value: "1" }, + }, + ], + }, + projector: { id: "gitlab.issue.created", version: 1, config: {} }, + activeFrom: "2000-01-01T00:00:00Z", + ...overrides, +}) + +describe("LocalEventingRuntime", () => { + it("records bounded normalization and projection outcomes without signal data", async () => + withDataDir(async (dataDir) => { + const observations: EventingTelemetryObservation[] = [] + const telemetry = { + record: (observation: EventingTelemetryObservation) => observations.push(observation), + } + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, telemetry) + runtime.activate(projection()) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 1) + + const malformed = structuredClone(gitlabIssueCreated) + firstLogRecord(malformed).attributes = firstLogRecord(malformed).attributes.filter( + ({ key }) => key !== "gitlab.project.path", + ) + strictEqual(runtime.evaluateOtlp("logs", malformed).failures.length, 1) + + const operationOutcomes = observations.map( + ({ operation, outcome }) => `${operation}:${outcome}`, + ) + ok(operationOutcomes.includes("normalization:success")) + ok(operationOutcomes.includes("projection:success")) + ok(operationOutcomes.includes("projection:failure")) + const serialized = JSON.stringify(observations) + strictEqual(serialized.includes("Wire GitLab events"), false) + strictEqual(serialized.includes("01K20GITLABISSUE42"), false) + strictEqual(serialized.includes("gitlab-issue-created"), false) + } finally { + store.close() + } + })) + + it("normalizes typed GitLab OTLP fields while preserving the existing warehouse encoding", () => { + const [signal] = normalizeOtlpLogs(gitlabIssueCreated, "2026-08-07T20:00:00Z") + strictEqual(signal?.occurrenceId, "01K20GITLABISSUE42") + strictEqual(signal?.identityQuality, "source") + strictEqual(signal?.source, "https://gitlab.example.test") + deepStrictEqual(signal?.fields.get("attribute:gitlab.issue.iid"), { + type: "int64", + value: "42", + }) + const batches = encodeLogs(gitlabIssueCreated) + strictEqual(batches.length, 1) + strictEqual(batches[0]?.rowCount, 1) + strictEqual(JSON.parse(batches[0]!.ndjson).log_attributes["gitlab.issue.iid"], "42") + }) + + it("uses the first nonblank occurrence alias and derives identity when every alias is blank", () => { + const aliased = structuredClone(gitlabIssueCreated) + const aliasedRecord = firstLogRecord(aliased) + aliasedRecord.attributes = [ + attr("event.id", { stringValue: " " }), + attr("cloudevents.id", { stringValue: " cloud-event-42 " }), + attr("gitlab.event.id", { stringValue: "gitlab-event-42" }), + ...aliasedRecord.attributes.filter( + ({ key }) => !["event.id", "cloudevents.id", "gitlab.event.id"].includes(key), + ), + ] + const [aliasedSignal] = normalizeOtlpLogs(aliased, "2026-08-07T20:00:00Z") + strictEqual(aliasedSignal?.occurrenceId, "cloud-event-42") + strictEqual(aliasedSignal?.identityQuality, "source") + + const derivedA = structuredClone(aliased) + const derivedARecord = firstLogRecord(derivedA) + derivedARecord.attributes = derivedARecord.attributes.map((entry) => + ["event.id", "cloudevents.id", "gitlab.event.id"].includes(entry.key) + ? attr(entry.key, { stringValue: entry.key === "event.id" ? "" : " \t " }) + : entry, + ) + const derivedB = structuredClone(derivedA) + firstLogRecord(derivedB).body = { stringValue: "A different issue occurrence" } + const [signalA] = normalizeOtlpLogs(derivedA, "2026-08-07T20:00:00Z") + const [signalB] = normalizeOtlpLogs(derivedB, "2026-08-07T20:00:00Z") + strictEqual(signalA?.identityQuality, "derived") + strictEqual(signalB?.identityQuality, "derived") + strictEqual(signalA?.occurrenceId?.startsWith("derived:sha256:"), true) + strictEqual(signalA?.occurrenceId === signalB?.occurrenceId, false) + }) + + it("catalogs only the scalar body field that the OTLP adapter can populate", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + throws( + () => + runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "text", type: "string" }, + }, + }), + ), + /unknown field body:text/, + ) + const activation = runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "value", type: "boolean" }, + }, + }), + ) + strictEqual(activation.spec.selector.op, "exists") + } finally { + store.close() + } + })) + + it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + strictEqual(runtime.hasActiveSource("otel.log"), false) + runtime.activate(projection()) + const first = runtime.evaluateOtlp("logs", gitlabIssueCreated) + strictEqual(first.failures.length, 0) + strictEqual(first.events.length, 1) + deepStrictEqual(first.events[0], { + specversion: "1.0", + id: first.events[0]!.id, + source: "https://gitlab.example.test", + type: "dev.maple.gitlab.issue.created.v1", + subject: "example/widgets/issues/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:gitlab-issue-created:v1", + tenantid: "local", + projectionid: "gitlab-issue-created", + projectionrevision: 1, + projectorid: "gitlab.issue.created", + projectorversion: 1, + sourceoccurrenceid: "01K20GITLABISSUE42", + sourceidentityquality: "source", + data: { + project: { id: "7", path: "example/widgets" }, + issue: { + id: "4200", + iid: "42", + title: "Wire GitLab events", + url: "https://gitlab.example.test/example/widgets/-/issues/42", + }, + actor: { id: "9", username: "operator" }, + serviceName: "gitlab-rails", + }, + }) + const staged = runtime.stage(first.events) + strictEqual(staged.inserted, 1) + strictEqual(runtime.listReady().events.length, 0) + deepStrictEqual( + runtime.listStaged().events.map(({ event }) => event), + first.events, + ) + const retry = runtime.evaluateOtlp("logs", gitlabIssueCreated) + strictEqual(retry.events[0]?.id, first.events[0]?.id) + strictEqual(runtime.stage(retry.events).deduplicated, 1) + runtime.markReady(staged.eventIds) + deepStrictEqual( + runtime.listReady().events.map(({ event }) => event), + first.events, + ) + deepStrictEqual(runtime.listStaged().events, []) + } finally { + store.close() + } + })) + + it("activates a validated revision without restart and reloads it after restart", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + let runtime = new LocalEventingRuntime(store) + runtime.activate(projection()) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 1) + runtime.activate( + projection({ + revision: 2, + selector: { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.closed" }, + }, + }), + ) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 0) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + try { + runtime = new LocalEventingRuntime(store) + strictEqual(runtime.listActive()[0]?.revision, 2) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 0) + } finally { + store.close() + } + })) + + it("does no normalization or event work for a source with no active projection", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + deepStrictEqual(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") }), { + events: [], + failures: [], + typeMismatchFields: [], + }) + } finally { + store.close() + } + })) +}) diff --git a/apps/cli/test/server-args.test.ts b/apps/cli/test/server-args.test.ts index b86f1d937..0021679c8 100644 --- a/apps/cli/test/server-args.test.ts +++ b/apps/cli/test/server-args.test.ts @@ -30,7 +30,10 @@ describe("local server bind host", () => { it("separates the bind address from the client-facing address", () => { strictEqual(resolveAdvertiseHost(undefined, undefined, "0.0.0.0"), "127.0.0.1") - strictEqual(resolveAdvertiseHost(undefined, " srvmini2.lan ", "0.0.0.0"), "srvmini2.lan") + strictEqual( + resolveAdvertiseHost(undefined, " node-a.example.test ", "0.0.0.0"), + "node-a.example.test", + ) strictEqual(resolveAdvertiseHost(" 192.0.2.10 ", "ignored", "0.0.0.0"), "192.0.2.10") strictEqual(resolveAdvertiseHost(" ", " [::1] ", "0.0.0.0"), "::1") }) @@ -70,7 +73,7 @@ describe("buildDetachedChildArgs", () => { const args = buildDetachedChildArgs({ entry: "/repo/apps/cli/src/bin.ts", host: "0.0.0.0", - advertiseHost: "srvmini2.lan", + advertiseHost: "node-a.example.test", port: 4318, dataDir: "/tmp/maple data", offline: true, @@ -84,7 +87,7 @@ describe("buildDetachedChildArgs", () => { "--host", "0.0.0.0", "--advertise-host", - "srvmini2.lan", + "node-a.example.test", "--port", "4318", "--data-dir", diff --git a/apps/cli/test/server-network.test.ts b/apps/cli/test/server-network.test.ts index 1e997074c..d9481e8b0 100644 --- a/apps/cli/test/server-network.test.ts +++ b/apps/cli/test/server-network.test.ts @@ -100,18 +100,23 @@ describe("local listener addresses", () => { }) describe("browser origin policy", () => { - const requestUrl = new URL("http://srvmini2.lan:4418/local/query") + const requestUrl = new URL("http://node-a.example.test:4418/local/query") const hostedOrigin = "https://local.maple.dev" - const browserHosts = ["srvmini2.lan", "127.0.0.1"] + const browserHosts = ["node-a.example.test", "127.0.0.1"] it("allows non-browser clients, the advertised same-origin UI, and the hosted UI", () => { strictEqual(isBrowserOriginAllowed(requestUrl, null, hostedOrigin, browserHosts), true) strictEqual( - isBrowserOriginAllowed(requestUrl, "http://srvmini2.lan:4418", hostedOrigin, browserHosts), + isBrowserOriginAllowed(requestUrl, "http://node-a.example.test:4418", hostedOrigin, browserHosts), true, ) strictEqual( - isBrowserOriginAllowed(requestUrl, "https://srvmini2.lan:4418", hostedOrigin, browserHosts), + isBrowserOriginAllowed( + requestUrl, + "https://node-a.example.test:4418", + hostedOrigin, + browserHosts, + ), true, ) strictEqual(isBrowserOriginAllowed(requestUrl, hostedOrigin, hostedOrigin, browserHosts), true) @@ -171,7 +176,8 @@ describe("browser origin policy", () => { deepStrictEqual(corsHeadersForAllowedOrigin(hostedOrigin), { "access-control-allow-origin": hostedOrigin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": "content-type, content-encoding, authorization", + "access-control-allow-headers": + "content-type, content-encoding, authorization, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", }) diff --git a/apps/local-ui/src/lib/constants.test.ts b/apps/local-ui/src/lib/constants.test.ts index cf150b2a5..54ad10873 100644 --- a/apps/local-ui/src/lib/constants.test.ts +++ b/apps/local-ui/src/lib/constants.test.ts @@ -19,9 +19,9 @@ describe("local UI endpoint selection", () => { }) it("keeps an embedded LAN or TLS-proxied UI same-origin", () => { - const page = location("https://srvmini2.lan:4418/?api_key=not-propagated") + const page = location("https://node-a.example.test:4418/?api_key=not-propagated") expect(localApiBaseForLocation(page)).toBe("") - expect(localOtlpEndpointForLocation(page)).toBe("https://srvmini2.lan:4418") + expect(localOtlpEndpointForLocation(page)).toBe("https://node-a.example.test:4418") }) it("keeps the Vite development UI same-origin for its proxied query and OTLP routes", () => { diff --git a/bun.lock b/bun.lock index cae3711bb..82722681d 100644 --- a/bun.lock +++ b/bun.lock @@ -52,12 +52,14 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/clickhouse-builder": "workspace:*", "@maple-dev/effect-sdk": "workspace:*", + "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", @@ -92,6 +94,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1", @@ -460,6 +463,18 @@ "effect": ">=4.0.0-beta.100 || >=4.0.0", }, }, + "packages/alerting-core": { + "name": "@maple/alerting-core", + "version": "0.0.0", + "dependencies": { + "@maple/eventing-core": "workspace:*", + }, + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/auth": { "name": "@maple/auth", "dependencies": { @@ -588,6 +603,19 @@ "typescript": "catalog:tooling", }, }, + "packages/eventing-core": { + "name": "@maple/eventing-core", + "version": "0.0.0", + "dependencies": { + "effect": "catalog:effect", + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/infra": { "name": "@maple/infra", "devDependencies": { @@ -1282,6 +1310,8 @@ "@maple/alerting": ["@maple/alerting@workspace:apps/alerting"], + "@maple/alerting-core": ["@maple/alerting-core@workspace:packages/alerting-core"], + "@maple/api": ["@maple/api@workspace:apps/api"], "@maple/auth": ["@maple/auth@workspace:packages/auth"], @@ -1306,6 +1336,8 @@ "@maple/email": ["@maple/email@workspace:packages/email"], + "@maple/eventing-core": ["@maple/eventing-core@workspace:packages/eventing-core"], + "@maple/infra": ["@maple/infra@workspace:packages/infra"], "@maple/ingest": ["@maple/ingest@workspace:apps/ingest"], diff --git a/docs/gitlab-event-projectors.md b/docs/gitlab-event-projectors.md new file mode 100644 index 000000000..34f9399ed --- /dev/null +++ b/docs/gitlab-event-projectors.md @@ -0,0 +1,386 @@ +# GitLab event projectors + +Status: version-1 producer contracts for the Maple Local eventing outbox. + +These pure projectors turn bounded, normalized GitLab OTLP log fields into +factual CloudEvents. They do not call GitLab, choose local routing policy, +create Matrix rooms, send Matrix messages, or acknowledge a downstream +consumer. Numeric `gitlab.project.id` is the durable project identity; mutable +project paths are display and subject metadata only. + +## Identity and compatibility + +The source receiver supplies a stable GitLab delivery UUID in `event.id` and +`gitlab.event.id`. When one delivery becomes multiple facts, their occurrence +IDs are `:`. The OTLP adapter preserves that +occurrence ID and Maple derives the CloudEvent ID from tenant, source kind, +source, occurrence ID, projection ID, and projection revision. Payload hashes +are audit data, not identity. + +Every new GitLab projector fails closed unless the normalized occurrence has +`sourceidentityquality: "source"` and a non-null source occurrence ID. This +prevents retries of one GitLab delivery from becoming distinct events. The +generic eventing adapter still supports deterministic derived identity for +other projector families; the restriction is intentionally at the new GitLab +projector boundary. + +New envelopes expose that preserved input as optional, backward-compatible +CloudEvents extensions `sourceoccurrenceid` and `sourceidentityquality`. +Historical envelopes lacking both fields remain schema-valid. A downstream +consumer can therefore persist the source occurrence ID, immutable Maple event +ID, and its own deterministic delivery transaction ID without parsing event +data. + +The original `gitlab.issue.created@1` projector and +`dev.maple.gitlab.issue.created.v1` output remain unchanged. It accepts the +OTLP LogRecord `eventName` field used by its existing fixture. New projectors +prefer the `event.name` attribute and retain the signal field as a compatibility +fallback. + +Complete output fixtures live in +`apps/cli/test/fixtures/gitlab-projectors.v1.json`. Their occurrence and +projection identity inputs are paired by index in +`apps/cli/test/fixtures/gitlab-projector-identities.v1.json`. + +Each factual GitLab `data` payload also has a checked-in JSON Schema under +`packages/eventing-core/schemas`. Fixture tests decode against the schema +selected by `dataschema`, and `schemas:check` fails when generated schemas +drift from the checked-in files. + +## Common input and safety contract + +Every new projector requires: + +- `event.name`: one explicit event name documented below; +- source-provided occurrence identity as described above; +- `gitlab.project.id`: positive OTLP int64; +- `gitlab.project.path`: non-blank bounded text. + +Common optional fields are `gitlab.project.old_path`, `gitlab.actor.id`, +`gitlab.actor.name`, `gitlab.actor.username`, `gitlab.event.result`, and +resource `service.name`. Every output payload contains +`project: { id, path, oldPath? }`, `sourceEvent`, the relevant factual object, +and any present actor, result, and service name. + +The projector boundary enforces: + +- scalar text at most 4 KiB after trimming; +- comment excerpts at most 1,024 UTF-8 bytes after removing control + characters and normalizing whitespace; +- canonical HTTP(S) URLs without credentials or query strings, at most 2,048 + UTF-8 bytes; comment URLs may retain a note-anchor fragment; +- positive object IDs and non-negative counts/durations; +- hexadecimal revisions, including all-zero before/after revisions; +- at most 20 failed-job summaries and no unknown summary fields; +- no raw payloads, variables, logs, full comments, arbitrary descriptions, or + secrets. + +Projector config objects are closed and currently have no fields, except the +unchanged legacy issue-created `includeBody` option. + +## Version-1 event vocabulary + +### Project lifecycle + +Projector: `gitlab.project.lifecycle@1` + +Type: `dev.maple.gitlab.project.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-project-lifecycle:v1` + +| `event.name` | Output `action` | +| -------------------------- | -------------------- | +| `project_create` | `created` | +| `project_update` | `updated` | +| `project_rename` | `renamed` | +| `project_transfer` | `transferred` | +| `project_archive` | `archived` | +| `project_unarchive` | `unarchived` | +| `project_deletion_request` | `deletion_requested` | +| `project_destroy` | `destroyed` | + +`gitlab.project.old_path` is optional and normally present for rename or +transfer facts. Transfer direction is deliberately not inferred here. + +### Issue lifecycle + +Projector: `gitlab.issue.lifecycle@1` + +Type: `dev.maple.gitlab.issue.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-issue-lifecycle:v1` + +| `event.name` | Output `issue.action` | +| -------------- | --------------------- | +| `issue_open` | `open` | +| `issue_update` | `update` | +| `issue_close` | `close` | +| `issue_reopen` | `reopen` | + +Required: `gitlab.issue.iid` and `gitlab.issue.type`. The type is derived from +Work Item Hook `object_attributes.type`, normalized to a lowercase snake token, +and bounded to 64 UTF-8 bytes. It is deliberately not an enum because GitLab +work-item types are configurable. Optional fields are positive +`gitlab.issue.id`, title, canonical URL, and bounded lowercase state. +Structured `gitlab.issue.labels` is an optional array of at most 50 non-blank +labels, each at most 256 UTF-8 bytes. The same issue object, including required +type and any labels, is used by issue-comment events. + +Project-scoped Work Item Hooks are normalized into this existing family: +create/open, update, close, and reopen facts use `issue_open`, `issue_update`, +`issue_close`, and `issue_reopen`; work-item notes use `issue_comment`. The +producer must place the normalized Work Item type in `gitlab.issue.type` and +must not expose the raw hook payload. Epics require group hooks and are outside +the current user-namespace project-hook coverage; this is an explicit external +coverage limit, not a second Maple event family. + +### Issue comments + +Projector: `gitlab.issue.comment@1` + +Type: `dev.maple.gitlab.issue.comment.v1` + +Schema: `urn:maple:event-schema:gitlab-issue-comment:v1` + +The only accepted event is `issue_comment`. Required fields are +`gitlab.issue.iid`, `gitlab.issue.type`, positive `gitlab.comment.id`, a +sanitized `gitlab.comment.excerpt`, and canonical `gitlab.comment.url`. +`gitlab.comment.system` remains optional because GitLab does not identify every +note source as a system note. The full comment body is never projected. + +### Merge-request lifecycle + +Projector: `gitlab.merge-request.lifecycle@1` + +Type: `dev.maple.gitlab.merge-request.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-merge-request-lifecycle:v1` + +| `event.name` | Output `mergeRequest.action` | +| ---------------------- | ---------------------------- | +| `merge_request_open` | `open` | +| `merge_request_update` | `update` | +| `merge_request_close` | `close` | +| `merge_request_reopen` | `reopen` | +| `merge_request_merge` | `merge` | +| `merge_request_review` | `review` | + +Required: positive `gitlab.merge_request.iid`, bounded +`gitlab.merge_request.title`, and canonical `gitlab.merge_request.url`. +Optional fields are source and target branches, hexadecimal +`gitlab.merge_request.commit`, and bounded `gitlab.merge_request.review_state`. +Review events require `review_state`. +Unlike the initial implementation, arbitrary `merge_request_` events +fail closed. + +### Merge-request comments + +Projector: `gitlab.merge-request.comment@1` + +Type: `dev.maple.gitlab.merge-request.comment.v1` + +Schema: `urn:maple:event-schema:gitlab-merge-request-comment:v1` + +`merge_request_comment` emits `comment.kind: "comment"` and +`merge_request_review_comment` emits `comment.kind: "review"`. Required fields +are `gitlab.merge_request.iid`, MR title and canonical URL, +`gitlab.comment.id`, sanitized excerpt, and canonical comment URL. The same +bounded comment sanitization rules as issue comments apply. + +### Completed pipelines + +Projector: `gitlab.pipeline.completed@1` + +Type: `dev.maple.gitlab.pipeline.completed.v1` + +Schema: `urn:maple:event-schema:gitlab-pipeline-completed:v1` + +The only accepted event is `ci_pipeline_completed`. Required fields are +positive `gitlab.ci.pipeline.id`, terminal +`gitlab.ci.pipeline.status: success|failed|canceled|skipped`, and canonical +`gitlab.ci.pipeline.url`. Optional scalar fields are: + +- `gitlab.ci.pipeline.iid`, `name`, `source`, and `detailed_status`; +- positive `gitlab.ci.pipeline.merge_request_iid`, emitted only when the + pipeline-to-MR association is unambiguous; +- `vcs.ref` and hexadecimal `vcs.ref.head.revision`; +- non-negative `duration_ms`, `queued_duration_ms`, and `stage_count`; +- for failed pipelines only, required non-negative `failed_job_count`, boolean + `failed_jobs_truncated`, and `failed_jobs` summary array. + +The structured OTLP attribute `gitlab.ci.pipeline.failed_jobs` is an array of +at most 20 objects. Each object permits only positive string-encoded int64 +`id`, bounded `name`, optional `stage`, literal status `failed`, and optional +canonical `url`. For failed pipelines, count and truncation metadata must agree +with the array length. Non-failed terminal pipelines reject failed-job metadata +rather than presenting a misleading empty failure summary. + +```json +{ + "pipeline": { + "id": "900", + "status": "failed", + "url": "https://gitlab.example.test/example/widgets/-/pipelines/900", + "failedJobCount": "3", + "failedJobsTruncated": true, + "failedJobs": [ + { + "id": "901", + "name": "unit", + "stage": "test", + "status": "failed", + "url": "https://gitlab.example.test/example/widgets/-/jobs/901" + } + ] + } +} +``` + +### Deployment lifecycle + +Projector: `gitlab.deployment.lifecycle@1` + +Type: `dev.maple.gitlab.deployment.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-deployment-lifecycle:v1` + +| `event.name` | Required `gitlab.deployment.status` | +| --------------------- | ----------------------------------- | +| `deployment_running` | `running` | +| `deployment_success` | `success` | +| `deployment_failed` | `failed` | +| `deployment_canceled` | `canceled` | +| `deployment_blocked` | `blocked` | +| `deployment_manual` | `manual` | + +Positive `gitlab.deployment.id`, bounded +`gitlab.deployment.environment`, matching status, hexadecimal +`gitlab.deployment.revision`, and canonical `gitlab.deployment.url` are all +required. + +### Job lifecycle + +Projector: `gitlab.job.lifecycle@1` + +Type: `dev.maple.gitlab.job.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-job-lifecycle:v1` + +The event is `ci_job_` and `gitlab.ci.job.status` must match. Allowed +statuses are `created`, `pending`, `preparing`, `waiting_for_resource`, +`running`, `success`, `failed`, `canceled`, `skipped`, `manual`, and +`scheduled`. Positive `gitlab.ci.job.id`, bounded `gitlab.ci.job.name`, and +status are required. Optional fields are stage, canonical job URL, positive +pipeline ID, ref/revision, non-negative duration, and boolean `allow_failure`. + +### Ref lifecycle + +Projector: `gitlab.ref.lifecycle@1` + +Type: `dev.maple.gitlab.ref.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-ref-lifecycle:v1` + +The production receiver normalizes raw `push`, `tag_push`, and +`repository_update` deliveries into exactly six deduplicated semantic facts: + +| `event.name` | `ref.type` | `ref.action` | +| --------------- | ---------- | ------------ | +| `branch_create` | `branch` | `create` | +| `branch_update` | `branch` | `update` | +| `branch_delete` | `branch` | `delete` | +| `tag_create` | `tag` | `create` | +| `tag_update` | `tag` | `update` | +| `tag_delete` | `tag` | `delete` | + +Required fields are bounded `vcs.ref` and hexadecimal +`vcs.ref.base.revision`/`vcs.ref.head.revision`. All-zero revisions are valid. +Canonical `gitlab.ref.url` and non-negative `gitlab.push.commit_count` are +optional because delete facts may not have a resolvable web URL and some hook +variants omit a commit count. Raw delivery names are not accepted by this +projector. + +### Release lifecycle + +Projector: `gitlab.release.lifecycle@1` + +Type: `dev.maple.gitlab.release.lifecycle.v1` + +Schema: `urn:maple:event-schema:gitlab-release-lifecycle:v1` + +`release_create`, `release_update`, and `release_delete` map to matching +`release.action` values. Bounded `gitlab.release.tag` is required. Positive +`gitlab.release.id`, bounded name, and canonical URL are optional because +delete hooks and older GitLab payload variants can omit them. Release +descriptions are intentionally excluded. + +Other optional fields follow the same source-fidelity rule: actor metadata can +be absent for system actions; issue global ID/title/state/URL and MR branch, +commit, or non-review review-state fields can be absent from some update or +delete hook forms; pipeline timing, ref, detailed status, and MR association can +be absent or ambiguous; failed-job `stage` and `url` can be absent from the +bounded lookup result; job pipeline/ref/timing/URL fields can be absent during +early lifecycle states. Projectors never synthesize these values. Tests cover +minimal valid events and reject omission of every field declared required. + +## Complete sample CloudEvent + +The checked-in fixture set contains one complete envelope for each output +family and enriched variants where useful. For example: + +```json +{ + "specversion": "1.0", + "id": "sha256:7f8aea4bc391d3b954fecd9ef5356b02aaeb4289ddfa25ef9c3c16217b0d77d1", + "source": "https://gitlab.example.test", + "type": "dev.maple.gitlab.deployment.lifecycle.v1", + "subject": "example/widgets/-/deployments/501", + "time": "2026-08-07T19:42:00.123456789Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", + "tenantid": "local", + "projectionid": "gitlab-deployment_failed", + "projectionrevision": 1, + "projectorid": "gitlab.deployment.lifecycle", + "projectorversion": 1, + "sourceoccurrenceid": "delivery-deployment:deployment_failed", + "sourceidentityquality": "source", + "data": { + "project": { "id": "42", "path": "example/widgets" }, + "deployment": { + "id": "501", + "environment": "production", + "status": "failed", + "revision": "a8123fa", + "url": "https://gitlab.example.test/example/widgets/-/deployments/501" + }, + "sourceEvent": "deployment_failed", + "result": "failure", + "serviceName": "gitlab-repository-events" + } +} +``` + +Downstream delivery may derive an idempotent Matrix transaction ID from the +immutable Maple CloudEvent ID. That transaction policy and Matrix's structured +`info.selfenrichment.maple` content are downstream consumer contracts and do +not belong in these projectors. + +## Producer handoff + +The receiver must emit the explicit event vocabulary and fields above before a +corresponding projection is activated. In particular, a producer upgrade is +needed wherever the current receiver does not yet emit issue/comment facts, +Work Item `object_attributes.type`, MR title/URL/review/comment facts, pipeline +canonical URL and bounded failed-job count/truncation summaries, deployment +identity/environment/status/revision/URL, job lifecycle fields, semantic ref +transitions, or release facts. For project-scoped Work Item Hooks the exact +handoff is: normalize `object_attributes.type` to lowercase snake case in +`gitlab.issue.type`; select one existing `issue_*` event name; emit the same +bounded issue/comment fields as an Issue Hook; and assign the indexed stable +source occurrence ID before OTLP export. The receiver owns webhook +normalization, duplicate semantic-transition suppression, source UUID indexing, +excerpt pre-sanitization, canonical URL construction, and failed-job +lookup/truncation. Maple validates and projects those facts but does not +reconstruct missing webhook data or call GitLab. Group-hook Epic coverage must +be implemented by the producer if that external scope is later authorized. diff --git a/docs/local-event-consumers.md b/docs/local-event-consumers.md new file mode 100644 index 000000000..f5e094d82 --- /dev/null +++ b/docs/local-event-consumers.md @@ -0,0 +1,129 @@ +# Maple Local event consumer protocol + +Status: version 1 durable downstream-consumer boundary for the Maple Local event outbox. + +This protocol lets a local bridge deliver ready Maple CloudEvents without destructive reads or a +second delivery database. It is intentionally transport-neutral: Maple does not send Matrix events, +store Matrix credentials, or choose rooms. + +## Credentials + +Maple creates two independent 32-byte hexadecimal credentials beside the configured data directory: + +- `.maintenance-token` administers projection and consumer configuration. +- `.event-consumer-token` permits only claim and acknowledgement requests. + +Both files must be real regular files. The consumer token is sent in +`x-maple-event-consumer-token`; it does not grant access to projection configuration, outbox +inspection, checkpoints, or retention controls. The existing maintenance token is sent in +`x-maple-maintenance-token` and cannot be substituted for the consumer token. + +## Consumer administration + +Consumer IDs match `^[a-z][a-z0-9._-]{0,63}$` and are unique. Disabled IDs remain reserved so an +operator cannot accidentally replace one consumer's durable position with an unrelated process. + +Register a consumer with the maintenance credential: + +```http +POST /local/eventing/consumers +Content-Type: application/json +X-Maple-Maintenance-Token: + +{"consumerId":"matrix","startAt":"beginning"} +``` + +`startAt` is exact: + +- `beginning` starts immediately before the earliest ready event still retained for the tenant. +- `latest` atomically skips every ready event visible at registration and receives later events. + +Successful registration returns `201` and the consumer record. Reusing any existing or disabled ID +returns `409`. `GET /local/eventing/consumers` lists records under maintenance authorization. + +Disable a consumer explicitly: + +```http +POST /local/eventing/consumers/disable +Content-Type: application/json +X-Maple-Maintenance-Token: + +{"consumerId":"matrix"} +``` + +Disabling clears any active lease and removes that cursor from the retention quorum. It does not +delete the audit record or permit the ID to be reused. + +## Claim and acknowledgement + +Claim between 1 and 1,000 ready events for a lease of 5 through 300 seconds: + +```http +POST /local/eventing/claims +Content-Type: application/json +X-Maple-Event-Consumer-Token: + +{"consumerId":"matrix","limit":100,"leaseSeconds":60} +``` + +A non-empty response has this shape: + +```json +{ + "consumerId": "matrix", + "leaseToken": "<64 lowercase hexadecimal characters>", + "leaseExpiresAt": "2026-08-13T16:01:00.000Z", + "throughSequence": 42, + "events": [ + { + "sequence": 42, + "event": { + "specversion": "1.0", + "id": "sha256:...", + "type": "dev.maple.gitlab.pipeline.completed.v1" + }, + "stagedAt": "2026-08-13T16:00:00.000Z", + "readyAt": "2026-08-13T16:00:00.010Z" + } + ] +} +``` + +The real `event` member is the complete validated CloudEvent. An empty claim returns null lease +fields and an empty event array. Only a SHA-256 hash of the lease token is stored. A second claim +while the lease is live returns `409`; at or after expiry it returns the same unacknowledged prefix, +possibly with a new token. + +After every event in the claimed batch has been accepted by the downstream system, acknowledge the +exact `throughSequence` returned by the claim: + +```http +POST /local/eventing/acks +Content-Type: application/json +X-Maple-Event-Consumer-Token: + +{"consumerId":"matrix","leaseToken":"","throughSequence":42} +``` + +Partial, extended, expired, missing, and wrong-token acknowledgements return `409`. Success returns: + +```json +{ "consumerId": "matrix", "acknowledgedThrough": 42, "prunedEvents": 0 } +``` + +Claims are at-least-once. A bridge crash after a downstream send and before acknowledgement causes +re-delivery after lease expiry. A Matrix bridge must therefore derive its Matrix transaction ID from +the immutable Maple CloudEvent `id`; retrying the same transaction ID makes that ambiguity harmless. + +## Retention, capacity, and checkpoints + +Ready events are eligible for pruning only through the lowest acknowledged sequence among all active +consumers for the tenant. Maple retains the newest 1,000 otherwise-prunable ready events by default. +Disabled consumers do not block pruning; staged events are never pruned by consumer acknowledgement. +If no consumer is active, acknowledgement retention performs no deletion. + +The eventing control database migrates transactionally from schema 1 to schema 2 on open. Existing +schema-1 control snapshots remain valid and are migrated after restore. Consumer cursors and leases +are part of the same SQLite backup as projection and outbox state. Consumer mutations enter the +server admission gate, so checkpoint exclusivity cannot capture a half-applied claim or +acknowledgement. diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md new file mode 100644 index 000000000..3a2a73a15 --- /dev/null +++ b/docs/signal-to-event-projection.md @@ -0,0 +1,922 @@ +# Signal-to-event projection architecture + +Status: implemented on `codex/issue-222-alerting-core`; downstream delivery remains out of scope + +Related work: [issue #222](https://github.com/MapleTechLabs/maple/issues/222), +`@maple/alerting-core` + +Audience: Maple maintainers and implementers of hosted or Maple Local runtimes + +## Decision summary + +Maple will treat immediate, per-occurrence event generation as an ingest concern, +not as a scheduled warehouse-query concern. + +- Each accepted OTLP record or provider webhook is decoded and normalized into a + typed signal once. +- An immutable snapshot of enabled signal projections is evaluated against that + signal before its scalar types are flattened for warehouse storage. +- Every matching projection invokes a registered, pure projector that produces a + factual [CloudEvents 1.0](https://github.com/cloudevents/spec/blob/main/cloudevents/spec.md) + event. +- Produced events enter a durable, idempotent outbox. Consumers and delivery + transports are downstream of that boundary. +- The original telemetry continues through the existing warehouse write path. +- chDB is not polled to discover newly arrived records. It remains the analytics + store and an optional, explicitly invoked replay source. +- Scheduled aggregate alerts remain query-driven. Alert lifecycle transitions + become another producer of typed events and use the same outbox as ingest-time + projections. + +The configurable matching model is a small, structured, typed predicate tree. It +is not arbitrary SQL and it is not a new textual expression language. The live +runtime evaluates the tree in memory. A warehouse adapter may lower the supported +subset to parameterized ClickHouse expressions for explicit historical replay, +but SQL behavior does not define the predicate semantics. + +## Problem + +Maple currently contains several mechanisms that are related but not expressed +through one event boundary: + +- hosted alert rules periodically query telemetry, update incident lifecycle + state, and request deliveries; +- PlanetScale receives signed webhooks and performs provider-specific work; +- Maple Local accepts OTLP records and writes them directly to chDB; +- future automation needs individual facts, such as a GitLab issue-created + signal, to become events that agents or other consumers can act on. + +Using the alert scheduler for the last case would give it the wrong semantics. +A windowed query answers a question about a set of stored records and normally +produces one aggregate observation. It cannot faithfully represent every +individual occurrence without cursors, overlap windows, late-arrival handling, +and deduplication. + +The current Local `logs` table has no ingestion sequence or native event ID. Its +sort key is designed for observability queries, and arbitrary OTLP attributes are +stored as strings. Repeatedly querying that table once per rule would therefore: + +- compete with ingest, UI queries, checkpoints, retention, and archive work; +- miss late records or repeatedly rediscover records unless a second deduplication + system is added; +- require casts that cannot always recover the source value's original type; +- turn an embedded analytical database into an inefficient message queue. + +The event layer is still useful. It belongs in front of chDB for live signals, +with chDB retained behind it for analytics and aggregate alert evaluation. + +## Goals + +1. Allow operators and integrations to configure which incoming signals become + typed events without writing SQL or changing core runtime code. +2. Evaluate each delivered signal in one ingest pass against all applicable + projections; do not issue one warehouse query per projection. +3. Preserve scalar types for string, boolean, integer, floating-point, + timestamp, and duration comparisons. +4. Make source adapters, projectors, event persistence, and consumers replaceable + behind explicit interfaces. +5. Give emitted events stable identities so retries do not create duplicate + logical events when the source provides stable occurrence identity. +6. Reuse the same event envelope and outbox for query-alert lifecycle events. +7. Keep Maple Local headless: matching and event persistence must work while no + browser is open. +8. Keep the core deterministic, bounded, tenant-scoped, and independent of a + database, network, scheduler, wall clock, or particular deployment host. + +## Non-goals + +- Adding NATS, JetStream, Kafka, or another general-purpose broker as a required + Maple component. +- Loading arbitrary third-party code into a running Maple process. A "plugin" in + this document is a compile-time registered module behind a stable interface. +- Defining sink delivery, Matrix behavior, agent authorization, or action policy. +- Replacing the Collector's routing, filtering, queueing, or authentication. +- Replacing scheduled queries for rates, percentiles, absence, threshold state, + or other aggregate alerts. +- Guaranteeing exactly-once external side effects across an uncooperative source, + Maple, and an arbitrary consumer. +- Automatically replaying old telemetry whenever a projection is created or + changed. +- Providing a general scripting language, joins, aggregation, arithmetic, + regular expressions, or user-provided SQL in the first version. + +## Terminology + +**Signal** +: One factual input occurrence after authentication, decoding, and normalization. +It may originate as an OTLP log/span/metric point or a provider webhook. + +**Source adapter** +: A module that verifies or accepts a source payload, normalizes occurrences into +typed signals, declares known fields, and supplies source identity when +available. + +**Signal projection** +: Durable configuration pairing a source kind, typed selector, and registered +projector. It says which source occurrences should be promoted into which +event representation. It is distinct from a downstream event subscription. + +**Selector** +: A bounded structured predicate over typed signal fields. + +**Projector** +: A pure, versioned function that maps one matching signal to a declared event +type and data schema. Provider-specific meaning belongs here rather than in the +eventing core. + +**Event** +: An immutable CloudEvents 1.0 envelope containing a typed factual payload. + +**Event outbox** +: Durable host storage that makes event creation idempotent and separates event +production from downstream delivery. + +**Event consumer** +: A downstream component interested in one or more event types. Webhooks, Matrix, +agents, issue creation, and existing provider responses are consumer concerns, +not selector or projector concerns. + +## Architecture + +There are two intentionally different event-production paths. They converge only +after a factual event has been produced. + +```mermaid +flowchart LR + Source["OTLP or provider source"] --> Gate["Authenticate / verify"] + Gate --> Decode["Decode once"] + Decode --> Signal["Typed normalized signal"] + + Signal --> Match["Ingest-time selector evaluation"] + Match --> Project["Registered signal projector"] + Project --> Outbox["Durable event outbox"] + + Signal --> Encode["Warehouse encoder"] + Encode --> Warehouse["chDB / hosted warehouse"] + + Warehouse --> Scheduled["Scheduled aggregate query"] + Scheduled --> Lifecycle["Alert evaluation and lifecycle"] + Lifecycle --> AlertProjector["Alert lifecycle projector"] + AlertProjector --> Outbox +``` + +The upper path handles occurrences such as "this GitLab issue was created". The +lower path handles conclusions such as "the error rate has remained above five +percent for ten minutes". Both can ultimately notify the same consumers without +pretending they have the same input or timing semantics. + +### Required module boundaries + +The architecture has four replaceable boundaries: + +1. **Source adapters** turn authenticated source payloads into typed signals. +2. **Selectors** determine whether a normalized signal qualifies. +3. **Projectors** map a qualifying signal to a typed factual event. +4. **Consumers** subscribe to event types downstream of the durable outbox. + +The eventing core owns the contracts and deterministic behavior. It does not know +about PlanetScale, GitLab, Matrix, chDB, PostgreSQL, Cloudflare Queues, or HTTP. + +PlanetScale is therefore one installed composition, not the model itself. Its +module can register a webhook source adapter and PlanetScale-specific projectors. +Those projectors can be replaced or supplemented without changing the selector +evaluator or downstream event contract. Existing PlanetScale behavior can later +be moved behind consumers of those typed events without putting provider actions +inside the projector. + +## Core data contracts + +The TypeScript below is illustrative. Canonical persisted encodings must be +defined with runtime schemas and shared conformance fixtures. + +### Typed values + +```ts +type SignalScalar = + | { readonly type: "string"; readonly value: string } + | { readonly type: "boolean"; readonly value: boolean } + | { readonly type: "int64"; readonly value: string } + | { readonly type: "float64"; readonly value: number } + | { readonly type: "timestamp"; readonly value: string } + | { readonly type: "duration"; readonly value: string } +``` + +`int64` and `duration` use decimal strings in serialized form so JavaScript does +not lose precision. Runtime evaluators may compile them to native `bigint` or the +equivalent host type. Timestamp values use canonical RFC 3339 with an explicit +offset in serialized form and compare as UTC instants. Duration values represent +integer nanoseconds. `float64` values must be finite; `NaN` and infinities are +rejected during normalization. + +Arrays and objects may be preserved for projector payloads, but selectors operate +only on declared scalar fields in version 1. + +### Normalized signal + +```ts +interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: unknown +} +``` + +- `sourceKind` chooses the compatible field catalog and projector registry. +- `source` is a stable URI identifying the logical producer or integration. +- `occurrenceId` is a source-issued stable identifier when one exists. +- `identityQuality: "source"` means the adapter expects the ID to survive source + retries and rebatching. `"derived"` identifies a canonical content fingerprint + with documented collision/collapse limitations. `"none"` cannot support a + durable once-only automation guarantee. +- `occurredAt` is source time; `observedAt` is Maple acceptance time. +- `fields` contains canonical built-ins and namespaced source attributes. It must + not contain secrets merely because they were present in the incoming payload. +- `data` is a bounded, schema-validated, source-specific representation available + to compatible projectors. It may contain arrays and objects that are not + selector-addressable, but it follows the adapter's redaction policy and is not + an unvalidated raw request body. + +The source adapter must not expose an unbounded raw payload as the selector field +space or projector input. + +### Field references and catalogs + +A selector uses logical field references, never physical column names: + +```ts +interface FieldRef { + readonly namespace: "signal" | "resource" | "scope" | "attribute" | "body" + readonly key: string + readonly type: SignalScalar["type"] +} +``` + +Each source adapter exposes a field catalog for known fields. A catalog entry +declares: + +- logical name and one or more scalar types; +- allowed selector operators; +- sensitivity and whether a projector may expose it by default; +- whether historical replay is `exact`, `coerced`, or `unavailable`; +- an optional backend-owned replay binding. This binding is not user SQL. + +OTLP resource, scope, and record attributes are open-ended. A projection may +reference an uncatalogued attribute by explicitly declaring its expected scalar +type. At runtime a differently typed value does not get coerced; it does not +match, and a bounded type-mismatch metric is recorded. Source-specific modules +should publish catalogs for common attributes so users do not need to repeat +those declarations. OTLP log bodies are deliberately closed in version 1: only +the polymorphic `body:value` field is selectable, and only when the entire body +is a scalar. Structured body objects and arrays remain available to projectors +through normalized signal data but do not advertise child selector fields that +the adapter cannot populate. + +### Selector AST + +```ts +type SignalPredicate = + | { readonly op: "all"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "any"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "not"; readonly clause: SignalPredicate } + | { readonly op: "exists"; readonly field: FieldRef } + | { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalLiteral + } + | { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalLiteral[] + } +``` + +Version 1 has the following semantics: + +| Operation | Supported types | Semantics | +| ------------------------ | ------------------------------------------- | ------------------------------------------------------------------------- | +| `exists` | all | True only when the field is present with a valid typed scalar. | +| `eq`, `neq` | all | Exact same-type comparison. A missing or mistyped field makes both false. | +| `gt`, `gte`, `lt`, `lte` | `int64`, `float64`, `timestamp`, `duration` | Ordered same-type comparison. | +| `contains` | `string` | Case-sensitive Unicode substring comparison. | +| `in` | all | Exact same-type membership; all literals must share the field type. | +| `all`, `any`, `not` | predicates | Total boolean composition with short-circuit evaluation. | + +There are no implicit casts. The string `"3"` is not the integer `3`; an integer +is not silently promoted to a float; and a string that resembles a date is not a +timestamp. Adapters may deliberately normalize a provider value into a declared +type, but that conversion is part of the source contract and is tested there. + +Missing values are not equivalent to null. Null source values are treated as +missing in version 1. Consequently `neq` requires a present field, whereas +`not(eq(...))` also matches a missing field. Configuration tooling should prefer +the explicit form that expresses the intended behavior. + +Validation happens before a projection can become active. Version 1 limits a +selector to: + +- nesting depth of 8; +- 64 total predicate nodes; +- 100 members in one `in` predicate; +- 4 KiB per string literal; +- no regular expressions, functions, arithmetic, joins, or user code. + +These bounds keep evaluation predictable and leave room for indexing active +projections by source kind and simple discriminating fields. `SignalScalar` +describes normalized source data and does not inherit the literal-only 4 KiB +limit; the OTLP adapter accepts source strings up to its separate 16 KiB bound. +The literal limit is normative in UTF-8 bytes. Because JSON Schema `maxLength` +counts characters rather than encoded bytes, the generated schema documents the +constraint and the shared multibyte conformance vectors enforce its exact edge. + +### Signal projection + +```ts +interface SignalProjectionSpec { + readonly id: string + readonly revision: number + readonly enabled: boolean + readonly tenantId: string + readonly sourceKind: string + readonly selector: SignalPredicate + readonly projector: { + readonly id: string + readonly version: number + readonly config: unknown + } + readonly activeFrom: string +} +``` + +Every semantic edit creates a new immutable revision. Activation is not +retroactive: the new revision sees signals accepted after the runtime atomically +installs its compiled registry snapshot. Historical processing requires an +explicit replay operation. + +The configuration record is data. Source adapters and projector implementations +are registered code. This is how matching remains configurable without making +authentication, provider semantics, or executable code user-supplied. + +For example, the first GitLab projection uses the following concrete contract: + +```json +{ + "id": "gitlab-issue-created", + "revision": 1, + "enabled": true, + "tenantId": "local", + "sourceKind": "otel.log", + "selector": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "signal", "key": "event.name", "type": "string" }, + "value": { "type": "string", "value": "gitlab.issue.created" } + }, + { + "op": "gte", + "field": { "namespace": "attribute", "key": "gitlab.issue.iid", "type": "int64" }, + "value": { "type": "int64", "value": "1" } + } + ] + }, + "projector": { "id": "gitlab.issue.created", "version": 1, "config": {} }, + "activeFrom": "2026-08-07T00:00:00Z" +} +``` + +The `gte` comparison above is an integer comparison, not lexicographic string +ordering. A timestamp predicate would similarly carry a `timestamp` literal and +compare normalized instants rather than formatted text. No query is generated +for either comparison on the live path. + +### Projector contract + +```ts +interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly validateConfig: (value: unknown) => ProjectorConfig + readonly project: (signal: NormalizedSignal, config: ProjectorConfig) => ProjectedEventData +} +``` + +A projector must be pure, deterministic, bounded, versioned, and free of I/O. It +does not create issues, call Matrix, send notifications, or mutate provider +state. It produces a factual event payload conforming to its declared schema. + +The registry may include a bounded generic field-mapping projector for +operator-defined factual events. Provider modules register semantic projectors +when field copying is insufficient. No runtime module loading is required. + +### Event envelope + +Produced events use CloudEvents 1.0 structured representation: + +```json +{ + "specversion": "1.0", + "id": "sha256:...", + "source": "urn:maple:source:otel:local", + "type": "dev.maple.gitlab.issue.created.v1", + "subject": "project/example/issues/42", + "time": "2026-08-07T19:42:00.000000000Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-issue:v1", + "tenantid": "...", + "projectionid": "...", + "projectionrevision": 3, + "data": {} +} +``` + +Names above are illustrative until the repository reserves its canonical event +type and schema namespace. + +The event ID is deterministic when stable occurrence identity exists: + +```text +SHA-256(tenant ID, source kind, source URI, occurrence ID, projection ID, projection revision) +``` + +The hash input uses a canonical length-delimited encoding, not string +concatenation. Projector version and output schema version are already fixed by +the immutable projection revision and must be recorded with the event. + +Sensitive source details belong in `data`, under the projector's explicit schema +and redaction policy. They must not be copied into CloudEvents context attributes, +logs, metrics labels, or idempotency keys. + +## Runtime behavior + +### Projection compilation and activation + +The host loads enabled projections for a tenant, validates them against the +source and projector registries, and compiles them into immutable predicate +functions. The active registry is swapped atomically. Every decoded ingest batch +uses exactly one registry snapshot, even if configuration changes while the batch +is being processed. + +The initial implementation may evaluate all projections in the applicable +`sourceKind` bucket. The registry may later index projections by exact-match +discriminators such as event name or service name. This is an optimization and +must not alter selector semantics or ordering. + +Projection evaluation is deterministic and side-effect free. All matching +projections run; this is not first-match routing. A signal may therefore produce +zero, one, or several different factual events. + +### Maple Local OTLP ingest + +Maple Local already decodes an OTLP request and then passes the decoded payload +to the warehouse encoder. The event seam belongs between those operations. + +The implementation should refactor decoding/normalization so that: + +1. the OTLP request is parsed once; +2. typed record values remain available to the matcher; +3. the existing warehouse rows are produced without changing their stored shape; +4. matched events are staged idempotently before ingest acknowledges success; +5. the telemetry insert completes; +6. staged events are marked ready for downstream consumption; +7. only then is the OTLP request acknowledged. + +When no projection matches, the path adds only bounded predicate work before the +existing chDB insert. + +If the event store cannot stage a required event, ingest returns a retryable +failure rather than silently losing automation. A source retry reuses the same +event ID when stable occurrence identity is available, so staging is idempotent. + +Staging and chDB insertion are not one transaction. A process crash after the +chDB insert but before the OTLP acknowledgement can still cause a duplicate raw +telemetry row on retry; that is already possible with at-least-once OTLP +delivery. The staged/ready outbox protocol prevents an event from becoming +dispatchable before the ingest attempt reaches its warehouse commit point. + +If atomic exactly-once storage across both systems later becomes a requirement, +the correct addition is a durable ingress journal before both writes. chDB +polling does not solve that problem. + +### Provider webhooks + +Provider authentication and replay protection run before normalization. The +host must establish a durable event boundary before acknowledging the provider. +The hosted PlanetScale route therefore projects a verified payload first and +enqueues the resulting CloudEvent together with its temporary parity payload; +the queue is its durable event boundary. + +The provider source adapter supplies the strongest available delivery or event +identity. It then uses the same selector, projector, event ID, and outbox +contracts as OTLP. Provider-specific response behavior does not live in the core; +it can be migrated behind consumers of the emitted event types. + +### Query-driven alerts + +Scheduled alert rules retain their existing execution model: + +1. the host schedules and claims a rule; +2. a warehouse query produces an aggregate `AlertObservation`; +3. `@maple/alerting-core` evaluates threshold and lifecycle state; +4. an alert lifecycle projector converts `trigger`, `resolve`, `renotify`, or + `test` intent into a CloudEvent; +5. the host persists it through the common event outbox. + +This path queries chDB or the hosted warehouse because its input is an aggregate +over time. It does not reuse the ingest-time signal selector, and the ingest-time +path does not impersonate an alert incident. + +### Historical replay + +Replay is an operator-invoked batch operation, never the live event mechanism. +It evaluates one projection revision over a bounded time range and must support a +dry-run count/sample mode before it can persist events. + +Every field catalog entry declares replay capability: + +- `exact`: stored data retains enough type and identity information to reproduce + live semantics; +- `coerced`: the adapter can apply an explicit cast, but the source type was lost + or identity is derived; +- `unavailable`: the backend cannot implement the live predicate faithfully. + +A replay request using a `coerced` field requires explicit operator +acknowledgement. A request using an unavailable field is rejected. The warehouse +compiler emits parameterized expressions through existing query-building +facilities; it never interpolates field names or literals supplied directly by a +user. + +Current Local OTLP attribute maps store strings, so arbitrary typed attributes +will generally be `coerced`, not `exact`. Replay event IDs are guaranteed to +deduplicate against live events only when the warehouse retained the same stable +source occurrence ID. + +## Processing and delivery guarantees + +The architecture uses precise, layered guarantees rather than the blanket phrase +"exactly once". + +| Boundary | Guarantee | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Source to Maple | At least once when the source/Collector retries; source-specific otherwise. | +| One accepted batch | One evaluation against one immutable projection-registry snapshot. | +| Projection with source-stable identity | Effectively-once event creation through deterministic ID plus unique outbox insertion. | +| Projection with derived identity | Best-effort deduplication; identical real occurrences may collapse and re-encoded retries may diverge. | +| Projection with no identity | At-least-once event creation only; durable automation should reject this configuration by default. | +| Outbox to consumer | At least once with an event ID/idempotency key; consumer-side external effects are outside this specification. | +| chDB telemetry row | Existing OTLP semantics; duplicate storage remains possible after ambiguous failures. | + +A projection intended to trigger external automation must require +`identityQuality: "source"` unless an operator explicitly accepts weaker +semantics. GitLab event instrumentation should therefore furnish a stable event +or delivery identifier as part of its source contract. + +## chDB responsibilities + +chDB is responsible for: + +- storing telemetry for interactive and analytical queries; +- serving scheduled aggregate-alert queries; +- serving bounded explicit replay where field capabilities allow it; +- participating in existing checkpoint, retention, and archive workflows. + +chDB is not responsible for: + +- acting as a live queue; +- maintaining one cursor per signal projection; +- deduplicating event delivery; +- storing mutable projection configuration or delivery attempts merely because + it stores the source telemetry; +- defining selector type semantics through ClickHouse casts. + +Version 1 requires no new column or sort-key change to the existing telemetry +tables. A future narrow event journal or ingress-identity column may improve +replay, but it must be justified separately and must not turn wide raw-telemetry +tables into queue state. + +## Alternatives considered + +| Alternative | Decision | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| One periodic chDB query per projection | Rejected. It repeats wide scans, introduces cursor/late-arrival problems, and competes with the analytical workload. | +| One shared query that tails all recent chDB rows | Rejected as the live path. It reduces query count but still lacks a reliable ingestion cursor and evaluates after scalar type loss. It may inform an explicit replay implementation. | +| ClickHouse materialized views per projection | Rejected. Mutable user configuration would become DDL, current attribute storage has already flattened types, and lifecycle/deduplication state still needs another store. | +| Collector OTTL as Maple's rule language | Kept as an optional deployment optimization. It is valuable for OTel-only routing but does not define provider-webhook behavior or Maple-managed dynamic configuration. | +| CEL as the first expression language | Deferred. CEL is safe and capable, but embedding compatible runtimes and defining warehouse lowering is more surface than the initial predicates require. Reconsider it if the bounded AST is demonstrably insufficient. | +| CloudEvents SQL as the signal selector | Rejected for raw signals. [CESQL 1.0](https://github.com/cloudevents/spec/blob/main/cesql/spec.md) filters CloudEvent context attributes but does not address arbitrary event `data`; it may be useful for downstream CloudEvent subscriptions. | +| NATS or another broker as the event abstraction | Rejected as a requirement. A broker can later implement an event transport port, but it does not replace source normalization, selector semantics, projectors, identity, or host persistence. | +| A custom textual DSL | Rejected. The structured predicate tree is the persisted intermediate representation; configuration UIs and APIs do not need a parser. | + +## Durable host ports + +The core needs interfaces rather than a prescribed database: + +```ts +interface SignalProjectionStore { + loadEnabled(tenantId: string): Promise +} + +interface EventOutboxStore { + stage(events: readonly CloudEvent[]): Promise + markReady(eventIds: readonly string[]): Promise +} +``` + +The real contracts also need revision/change notification, unique event IDs, +bounded batch operations, health inspection, and recovery of staged records. + +Hosted Maple may implement these ports with its relational state and queue +infrastructure. Maple Local needs a small transactional control-state store whose +rules, outbox, and migration identity survive restart. That state is not covered +by chDB checkpoints automatically; backup, restore, and schema migration are part +of the Local host adapter's acceptance criteria. + +The physical Local store is an implementation decision, but it must provide: + +- uniqueness on event ID; +- atomic projection revision writes; +- atomic event staging and readiness transitions; +- bounded recovery of stranded staged events; +- crash-safe migrations and explicit backup/restore behavior; +- no dependency on a browser process. + +## Package and host ownership + +The intended ownership is: + +- `packages/eventing-core` (new): language-neutral schemas, selector validation, + the reference TypeScript evaluator, projector registry contracts, canonical + event identity, and conformance fixtures. No database, network, scheduler, or + global clock dependencies. +- `packages/alerting-core` (existing): aggregate alert evaluation and incident + lifecycle. It remains distinct and later emits through an eventing-core port. +- `packages/domain`: public/API schemas when projection CRUD becomes public. +- `apps/cli`: Maple Local OTLP source adapter, compiled-registry lifecycle, + durable Local ports, ingest staging, and optional replay adapter. +- `apps/api`: provider webhook adapters and hosted persistence wiring. +- `apps/ingest`: a future Rust OTLP adapter only when hosted per-signal projection + is required. + +The canonical JSON schemas and fixture corpus, rather than TypeScript source +types, define cross-language behavior. A Rust implementation must pass the same +valid/invalid selector cases, typed comparison cases, canonical event-ID vectors, +and projection fixtures before it can claim compatibility. + +[OpenTelemetry Transformation Language](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl) +can remain a Collector-side optimization or adapter. It is not the universal +Maple contract because it is coupled to OTel Collector contexts and does not +cover provider webhooks. [CEL](https://cel.dev/overview/cel-overview) is the +preferred language to reconsider if real requirements outgrow the bounded AST; +version 1 does not embed CEL runtimes or define a CEL-to-ClickHouse compiler. + +## Security and tenancy + +- Authentication or provider signature verification occurs before a source + adapter may produce a signal. +- Every signal, projection, event, and outbox operation carries an explicit + tenant ID. Cross-tenant registry lookup or event fanout is forbidden. +- User configuration cannot name SQL columns, inject SQL fragments, load code, + call functions, or select secrets outside the source field catalog. +- Source adapters mark sensitive fields. Generic projectors exclude them by + default; provider projectors must opt in deliberately and document why. +- Projected event size and source-field size are bounded before outbox insertion. +- Runtime errors and telemetry must not record full sensitive payloads. +- Sink URL validation, private-network policy, signing, and agent authorization + remain downstream policies. General eventing must not weaken hosted SSRF + protections. + +## Failure handling and observability + +Malformed projection configuration is rejected before activation. The reference +evaluator is total: missing fields and runtime type mismatches produce defined +non-matches rather than exceptions. + +A projector must return either schema-valid event data or a bounded typed +projection failure. A bad occurrence must not create an infinite source retry +loop. The host records the failure against projection ID/revision and occurrence +identity, exposes degraded health, and quarantines or dead-letters according to a +bounded policy. Exact quarantine policy belongs to the host adapter, but silently +dismissing a durable projection failure is not allowed. + +Required low-cardinality telemetry includes: + +- received signals by source kind; +- selector evaluations and matches by projection ID; +- selector type mismatches by source kind and field catalog key; +- projection failures; +- outbox staged, deduplicated, ready, and stranded counts; +- evaluation and staging latency; +- active projection count and registry revision; +- replay scanned, matched, emitted, and deduplicated counts. + +Raw field values, subjects, event IDs, and arbitrary event types must not become +unbounded metric labels. + +Maple Local implements the ingest-time subset as +`maple.eventing.operations_total`, `maple.eventing.operation_duration_ms`, and +`maple.eventing.consumer_lag_events`. Their only attributes are bounded +`operation`, `outcome`, and `source_kind` values. The operations cover +normalization, projection success/failure, outbox stage/ready/deduplication, and +consumer claim/ack/lease/lag. Tenant IDs, projection and consumer IDs, event +types and IDs, URLs, payload fields, lease tokens, and credentials are never +metric attributes. Replay and stranded-outbox telemetry remain applicable only +when those optional host operations run. + +## Compatibility and migration + +This design extends rather than replaces the host-neutral alert-core extraction +already on the issue-222 branch. + +1. Existing hosted aggregate alerts continue using their scheduler, query, + lifecycle, and delivery behavior while the event contract is introduced. +2. The new eventing core lands without runtime activation and with conformance + fixtures. +3. Maple Local adds ingest-time projection behind an explicit feature/config + gate. With no active projections, observable ingest and chDB behavior remain + unchanged. +4. A GitLab issue-created OTLP fixture proves the end-to-end source identity, + typed selector, projector, retry deduplication, and durable outbox path. +5. PlanetScale is adapted behind the same source/projector interfaces while its + existing externally visible behavior remains intact. A compare/dual-observe + period should precede removal of direct hard-coded handling. +6. Alert lifecycle intents are projected into the same CloudEvents/outbox model + after parity tests show no change to trigger, resolve, renotify, test, + suppression, or retry semantics. +7. Warehouse replay is added only after the live path is proven and replay + capability metadata is implemented. + +No migration step requires NATS, a per-rule chDB cursor, or a new raw-telemetry +sort key. + +## Implementation slices for the next goal + +### Slice 1 — Contract and evaluator + +- Add `packages/eventing-core`. +- Define runtime schemas for typed values, fields, predicates, projection specs, + projector registrations, and CloudEvent output. +- Implement validation, compilation, and the pure reference evaluator. +- Add canonical JSON and event-ID test vectors. +- Add complexity-limit and hostile-input tests. + +### Slice 2 — Local durable control state + +- Select and document the Local transactional store. +- Implement projection revision and outbox ports, migrations, recovery, and + backup/restore hooks. +- Expose headless health inspection before UI work. + +### Slice 3 — Local ingest seam + +- Refactor OTLP normalization to preserve typed values without decoding twice. +- Load and atomically swap compiled projection snapshots. +- Stage matching events, insert telemetry, mark events ready, and acknowledge. +- Prove that the live path executes no chDB `SELECT` and adds no scheduler. + +### Slice 4 — First vertical: GitLab event to durable Maple event + +- Capture the real GitLab OTLP field contract and stable occurrence identity. +- Register its field catalog and issue-event projector. +- Configure an issue-created projection without hard-coded selector values. +- Verify duplicate source deliveries create one logical outbox event. + +This slice stops at the outbox. Matrix and agent-action behavior are a downstream +goal using the produced typed event. + +### Slice 5 — Existing producer convergence + +- Adapt PlanetScale webhook inputs to the source/projector contracts. +- Project alert lifecycle intents into CloudEvents. +- Preserve existing provider and alert behavior with parity fixtures before + switching consumers. + +### Slice 6 — Optional replay + +- Add per-field replay capability declarations. +- Implement bounded dry-run and explicit emission modes. +- Add evaluator-versus-ClickHouse conformance tests for every `exact` binding. + +## Acceptance criteria + +The first usable implementation is complete when all of the following are true: + +1. A configured GitLab issue-created OTLP signal is matched before chDB encoding + and produces a schema-valid CloudEvent while the telemetry record is still + stored normally. +2. Re-delivery of a source-stable occurrence produces the same event ID and one + logical outbox record. +3. A nonmatching signal performs no warehouse read and creates no event. +4. Several active projections are evaluated from one registry snapshot, and all + matches run. +5. Integer, float, timestamp, duration, boolean, and string truth-table fixtures + pass with no implicit coercion. +6. Projection changes are validated, revisioned, persisted, and activated + atomically without restarting Maple Local. +7. Rules and ready/staged outbox records survive process restart and participate + in documented backup and recovery. +8. chDB query alerts retain their existing aggregate and lifecycle behavior. +9. No implementation requires a browser, a new broker, arbitrary runtime code, + raw SQL configuration, or a per-projection chDB poller. +10. The event envelope and selector fixture corpus are sufficient for a second + language implementation to demonstrate semantic parity. + +## Settled implementation choices + +The TypeScript reference implementation settles the remaining host choices as +follows: + +- Maple Local stores projection revisions, failures, and the staged/ready outbox + in SQLite at `/control/eventing.sqlite`, using WAL and `synchronous = +FULL`. While ingest is quiesced, backup first completes and verifies a blocking + `wal_checkpoint(TRUNCATE)` so the serialized database contains every committed + control-store transaction rather than only the main SQLite file. A version-2 + Maple checkpoint contains `control.sqlite` beside the chDB + backup and binds its byte count, SHA-256 digest, schema version, and row counts + in the checkpoint manifest. Version-1 checkpoints remain readable and restore + an empty control store. +- The first GitLab instrumentation contract is an OTLP log whose LogRecord + `eventName` is `gitlab.issue.created`. `event.id` is the preferred stable + source occurrence identifier; `cloudevents.id` and `gitlab.event.id` are + accepted aliases. The semantic projector requires `gitlab.project.path` and + integer `gitlab.issue.iid`; it also recognizes project/issue IDs, title, URL, + and actor attributes. GitLab itself does not synthesize this contract merely + because Maple is running: the emitting instrumentation or adapter must attach + those fields. +- Maple-owned event types use `dev.maple.*.v1`; schemas use + `urn:maple:event-schema:*:v1`. The current vertical emits + `dev.maple.gitlab.issue.created.v1` with + `urn:maple:event-schema:gitlab-issue-created:v1`. +- Attribute strings are limited to 16 KiB, source/event identities to 256 + characters (long stable inputs are represented by a SHA-256 URN), each + attribute namespace to 256 entries, + nested values to depth 8 and 1,024 nodes, normalized source data to 256 KiB, + and a canonical outbox CloudEvent to 256 KiB. Secret-like attribute names are + excluded from the projection field and data views. +- The Local TypeScript path is the reference live implementation. Hosted Rust + ingest remains a later adapter and must pass the shared schemas and fixture + corpus before claiming parity. +- Verified non-test PlanetScale webhooks run through a registered + `planetscale.webhook` source adapter, selector, and projector before the route + acknowledges them. The dedicated Cloudflare Queue durably carries + `dev.maple.planetscale.webhook.received.v1`; its temporary provider payload + keeps the existing issue and timeline consumers behaviorally unchanged while + they migrate to the event contract. Queue consumers accept both the new + event-bearing message and the exact pre-migration message shape, reconstructing + the deterministic event from the older message's durable fields during rolling + upgrades. +- Hosted query-alert delivery rows remain that producer's durable outbox. Their + payload now includes an additive deterministic + `dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` CloudEvent while + retaining every legacy top-level delivery field. Retry creation preserves the + originally stored JSON, including the CloudEvent ID and future additive fields, + instead of round-tripping it through a lossy legacy schema. +- Historical replay execution remains deliberately unimplemented in this + change. Field catalogs already declare `exact`, `coerced`, or `unavailable`, + but Local's current arbitrary attribute maps have lost source scalar type and + its warehouse rows do not furnish a native occurrence ID. A later bounded, + operator-invoked replay adapter must require explicit coercion acknowledgement + and pass live-evaluator conformance tests; the live path never falls back to a + chDB poller in the meantime. +- Projector failures with a source occurrence ID are idempotent per projection + revision. Local retains a bounded newest 10,000 failure rows per tenant and + exposes the count through the authenticated headless health endpoint. A + projector failure does not retry a valid telemetry occurrence forever; + infrastructure failure to persist required state remains retryable. + +Maple Local activates immutable revisions with authenticated +`POST /local/eventing/projections`. The same maintenance credential protects +`GET /local/eventing/projections`, `/local/eventing/health`, +`/local/eventing/outbox`, and consumer administration. Ready records receive a separate, append-only +readiness `sequence` on their first staged-to-ready transition; +`?after=&limit=` therefore cannot skip an older staged event that is +recovered after newer events were already read. `?state=staged` uses the original +staging sequence for bounded inspection of records stranded before the chDB +commit point. The Local store defaults to at most 10,000 events and 256 MiB of +canonical event JSON. Staging fails closed with a retryable ingest error before +either cap can be exceeded. Inspection remains non-destructive. Named downstream +consumers use the separate [Maple Local event consumer protocol](./local-event-consumers.md) for +leased, at-least-once claims and exact whole-batch acknowledgement. Ready-event pruning advances only +through the slowest active consumer and retains a bounded acknowledged tail; staged events are never +pruned by delivery acknowledgement. + +Re-delivery is the safe recovery operation: it deduplicates the same staged event +ID and promotes it only after the warehouse write succeeds. Maple never blindly +promotes an old staged record because, after a crash, the control store alone +cannot prove whether the corresponding chDB write committed. Activation requires +authentication, a bounded request body, structural budget validation, and full +registry compilation before acquiring global quiescence. Only the projection +revision commit and immutable runtime-registry swap occur while ingest is +quiesced, so invalid credentials, incomplete bodies, and expensive validation do +not close admission and every ingest request still observes exactly one registry +version. Concurrent maintenance requests receive an intentional conflict response. diff --git a/packages/alerting-core/README.md b/packages/alerting-core/README.md new file mode 100644 index 000000000..0571b49e0 --- /dev/null +++ b/packages/alerting-core/README.md @@ -0,0 +1,40 @@ +# `@maple/alerting-core` + +Host-neutral alert evaluation and incident-lifecycle semantics shared by Maple +deployment targets. + +The core is deliberately free of database, telemetry warehouse, scheduler, +network, and wall-clock dependencies. A host supplies observations and durable +state, calls the pure decision functions, then applies the returned transition +and delivery intent through its own adapters. + +Current hosted adapters live in `apps/api` and are scheduled by +`apps/alerting`. A Maple Local adapter can use the same core with chDB-backed +queries, Local durable state, an in-process scheduler, and its own outbound URL +policy without importing either hosted application. + +This package covers scheduled aggregate alerts. Immediate per-occurrence events +use the separate ingest-time architecture described in +[`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md). +Both paths may ultimately publish through the same typed event outbox, but raw +signal matching does not poll chDB or impersonate an alert lifecycle. + +The boundary is: + +- query adapter -> `AlertObservation`; +- evaluation policy + observation -> `AlertEvaluation`; +- persistence snapshot + evaluation -> `AlertLifecyclePlan`; +- host persists the plan, projects its optional `eventType` into the common + CloudEvents envelope, and sends that event through a delivery adapter; +- delivery adapters share idempotency-key and bounded retry policy helpers; +- host clock supplies `nowMs`; the core never reads global time. + +Rule CRUD, storage schemas, scheduler claims, destination configuration, and +delivery transports remain host concerns. This keeps Local UI work optional: +the alert runtime can evaluate and deliver while no browser is open. + +Hosted alert delivery rows are the existing durable outbox for this producer. +Their additive `event` payload contains the deterministic +`dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` envelope; current +destinations continue to receive the legacy top-level payload fields during the +migration. diff --git a/packages/alerting-core/package.json b/packages/alerting-core/package.json new file mode 100644 index 000000000..9c7034969 --- /dev/null +++ b/packages/alerting-core/package.json @@ -0,0 +1,21 @@ +{ + "name": "@maple/alerting-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@maple/eventing-core": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/alerting-core/src/index.test.ts b/packages/alerting-core/src/index.test.ts new file mode 100644 index 000000000..f0786e57a --- /dev/null +++ b/packages/alerting-core/src/index.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it } from "vitest" +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + projectAlertLifecycleEvent, + type AlertEvaluation, +} from "./index" + +const breached: AlertEvaluation = { + status: "breached", + value: 11, + sampleCount: 5, + threshold: 10, + thresholdUpper: null, + comparator: "gt", + reason: "above threshold", + derivedFromNoData: false, +} + +const healthy: AlertEvaluation = { ...breached, status: "healthy", value: 9 } + +const policy = { + consecutiveBreachesRequired: 2, + consecutiveHealthyRequired: 2, + renotifyIntervalMinutes: 10, +} + +describe("evaluateAlertObservation", () => { + it("applies thresholds and rounds weighted sample counts", () => { + expect( + evaluateAlertObservation( + { + comparator: "between", + threshold: 10, + thresholdUpper: 20, + minimumSampleCount: 2, + noDataBehavior: "skip", + }, + { value: 15, sampleCount: 2.4, hasData: true }, + "inside range", + ), + ).toMatchObject({ status: "breached", sampleCount: 2, reason: "inside range" }) + }) + + it("marks a zero synthesized from no data so lifecycle resolution can fail closed", () => { + expect( + evaluateAlertObservation( + { + comparator: "gt", + threshold: 10, + thresholdUpper: null, + minimumSampleCount: 0, + noDataBehavior: "zero", + }, + { value: null, sampleCount: 0, hasData: false }, + "above threshold", + ), + ).toMatchObject({ status: "healthy", value: 0, derivedFromNoData: true }) + }) +}) + +describe("planAlertLifecycle", () => { + it("opens only after the configured breach count", () => { + const first = planAlertLifecycle({ + policy, + evaluation: breached, + state: null, + openIncident: null, + nowMs: 1_000, + }) + expect(first).toMatchObject({ transition: "none", state: { consecutiveBreaches: 1 } }) + + const second = planAlertLifecycle({ + policy, + evaluation: breached, + state: first.state, + openIncident: null, + nowMs: 2_000, + }) + expect(second).toMatchObject({ transition: "opened", eventType: "trigger" }) + }) + + it("suppresses a flapping trigger and its matching resolve", () => { + const opened = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 1, consecutiveHealthy: 0 }, + openIncident: null, + nowMs: 600_000, + previousNotificationAtMs: 300_000, + }) + expect(opened).toMatchObject({ + transition: "opened", + eventType: null, + notificationSuppression: "flapping", + inheritedNotificationAtMs: 300_000, + }) + + const resolved = planAlertLifecycle({ + policy, + evaluation: healthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 600_000, + lastNotifiedAtMs: opened.inheritedNotificationAtMs, + lastDeliveredEventType: null, + }, + nowMs: 700_000, + }) + expect(resolved).toMatchObject({ + transition: "resolved", + eventType: null, + notificationSuppression: "flap_resolution", + }) + }) + + it("advances the notification anchor when renotify becomes due", () => { + const plan = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 2, consecutiveHealthy: 0 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 1_000, + lastDeliveredEventType: "trigger", + }, + nowMs: 601_000, + }) + expect(plan).toMatchObject({ + transition: "continued", + eventType: "renotify", + advanceNotificationAnchor: true, + }) + }) + + it("holds a no-data recovery until the host proves telemetry liveness", () => { + const noDataHealthy = { ...healthy, derivedFromNoData: true } + const input = { + policy, + evaluation: noDataHealthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 0, + lastDeliveredEventType: "trigger" as const, + }, + nowMs: 1_000, + } + expect(planAlertLifecycle(input)).toMatchObject({ transition: "none", hold: "missing_telemetry" }) + expect(planAlertLifecycle({ ...input, allowNoDataResolution: true })).toMatchObject({ + transition: "resolved", + eventType: "resolve", + }) + }) +}) + +describe("interleaveAlertRulesByTenant", () => { + it("preserves each tenant's order while round-robining tenants", () => { + const rows = [ + { tenantId: "a", id: "a1" }, + { tenantId: "a", id: "a2" }, + { tenantId: "b", id: "b1" }, + { tenantId: "a", id: "a3" }, + { tenantId: "b", id: "b2" }, + ] + expect(interleaveAlertRulesByTenant(rows, (row) => row.tenantId).map(({ id }) => id)).toEqual([ + "a1", + "b1", + "a2", + "b2", + "a3", + ]) + }) +}) + +describe("delivery policy", () => { + it("projects lifecycle intents into deterministic common CloudEvents", () => { + const input = { + tenantId: "org-1", + ruleId: "rule-1", + ruleName: "High errors", + incidentId: "incident-1", + eventType: "trigger" as const, + incidentStatus: "open", + groupKey: "checkout", + signalType: "error_rate", + severity: "critical", + comparator: "gt" as const, + threshold: 5, + thresholdUpper: null, + windowMinutes: 5, + value: 7.2, + sampleCount: 12, + occurredAtMs: 1_786_131_720_123, + } + const event = projectAlertLifecycleEvent(input) + expect(event).toEqual(projectAlertLifecycleEvent(input)) + expect(event).toMatchObject({ + type: "dev.maple.alert.lifecycle.trigger.v1", + subject: "alert-incidents/incident-1", + tenantid: "org-1", + projectionid: "alert-lifecycle", + data: { eventType: "trigger", incidentId: "incident-1" }, + }) + expect(() => projectAlertLifecycleEvent({ ...input, occurredAtMs: Number.MAX_SAFE_INTEGER })).toThrow( + "outside the supported date range", + ) + }) + + it("builds stable idempotency keys", () => { + expect(makeAlertDeliveryKey("incident", "destination", "trigger", 42)).toBe( + "incident:destination:trigger:42", + ) + }) + + it("caps exponential retry delay and attempts", () => { + expect(alertDeliveryRetryDelayMs(1, 123)).toBe(60_123) + expect(alertDeliveryRetryDelayMs(5, 999)).toBe(900_999) + expect(canRetryAlertDelivery(4, true)).toBe(true) + expect(canRetryAlertDelivery(5, true)).toBe(false) + expect(canRetryAlertDelivery(1, false)).toBe(false) + }) +}) diff --git a/packages/alerting-core/src/index.ts b/packages/alerting-core/src/index.ts new file mode 100644 index 000000000..6d5b523d9 --- /dev/null +++ b/packages/alerting-core/src/index.ts @@ -0,0 +1,404 @@ +import { makeCloudEvent, type MapleCloudEvent } from "@maple/eventing-core" + +export type AlertComparator = "gt" | "gte" | "lt" | "lte" | "eq" | "neq" | "between" | "not_between" + +export type AlertEvaluationStatus = "breached" | "healthy" | "skipped" + +export interface AlertObservation { + readonly value: number | null + readonly sampleCount: number + readonly hasData: boolean +} + +export interface AlertEvaluationPolicy { + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly minimumSampleCount: number + readonly noDataBehavior: "skip" | "zero" +} + +export interface AlertEvaluation { + readonly status: AlertEvaluationStatus + readonly value: number | null + readonly sampleCount: number + readonly threshold: number + readonly thresholdUpper: number | null + readonly comparator: AlertComparator + readonly reason: string + /** A healthy result derived from an empty window synthesized as zero. */ + readonly derivedFromNoData: boolean +} + +export const compareAlertThreshold = ( + value: number, + comparator: AlertComparator, + threshold: number, + thresholdUpper: number | null = null, +): boolean => { + switch (comparator) { + case "gt": + return value > threshold + case "gte": + return value >= threshold + case "lt": + return value < threshold + case "lte": + return value <= threshold + case "eq": + return value === threshold + case "neq": + return value !== threshold + case "between": + return thresholdUpper != null && value >= threshold && value <= thresholdUpper + case "not_between": + return thresholdUpper != null && (value < threshold || value > thresholdUpper) + } +} + +export const evaluateAlertObservation = ( + policy: AlertEvaluationPolicy, + observation: AlertObservation, + reason: string, +): AlertEvaluation => { + // Sample-weighted counts can be fractional while durable alert state commonly + // stores an integer. Normalize at the host-neutral boundary. + const sampleCount = Math.round(observation.sampleCount) + const value = observation.hasData ? observation.value : policy.noDataBehavior === "zero" ? 0 : null + + if (!observation.hasData && policy.noDataBehavior === "skip") { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "No data in the selected window", + derivedFromNoData: false, + } + } + + if (sampleCount < policy.minimumSampleCount) { + return { + status: "skipped", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: `Sample count ${sampleCount} is below minimum ${policy.minimumSampleCount}`, + derivedFromNoData: false, + } + } + + if (value == null) { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "Alert evaluation did not return a scalar value", + derivedFromNoData: false, + } + } + + return { + status: compareAlertThreshold(value, policy.comparator, policy.threshold, policy.thresholdUpper) + ? "breached" + : "healthy", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason, + derivedFromNoData: !observation.hasData, + } +} + +export interface AlertLifecyclePolicy { + readonly consecutiveBreachesRequired: number + readonly consecutiveHealthyRequired: number + readonly renotifyIntervalMinutes: number +} + +export interface AlertLifecycleState { + readonly consecutiveBreaches: number + readonly consecutiveHealthy: number +} + +export interface AlertLifecycleIncident { + readonly firstTriggeredAtMs: number + readonly lastNotifiedAtMs: number | null + readonly lastDeliveredEventType: AlertEventType | null +} + +export type AlertEventType = "trigger" | "resolve" | "renotify" | "test" +export type AlertIncidentTransition = "none" | "opened" | "continued" | "resolved" +export type AlertNotificationSuppression = "flapping" | "flap_resolution" | null +export type AlertLifecycleHold = "missing_telemetry" | null + +export interface AlertLifecycleEventInput { + readonly tenantId: string + readonly ruleId: string + readonly ruleName: string + readonly incidentId: string | null + readonly eventType: AlertEventType + readonly incidentStatus: string + readonly groupKey: string | null + readonly signalType: string + readonly severity: string + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly windowMinutes: number + readonly value: number | null + readonly sampleCount: number | null + readonly occurredAtMs: number +} + +/** Project a query-alert lifecycle intent into the common factual event envelope. */ +export const projectAlertLifecycleEvent = (input: AlertLifecycleEventInput): MapleCloudEvent => { + if (!Number.isSafeInteger(input.occurredAtMs) || input.occurredAtMs < 0) + throw new Error("alert lifecycle event time must be a non-negative epoch millisecond") + const occurredAtDate = new Date(input.occurredAtMs) + if (Number.isNaN(occurredAtDate.getTime())) + throw new Error("alert lifecycle event time is outside the supported date range") + const occurredAt = occurredAtDate.toISOString() + const occurrenceId = `${input.incidentId ?? input.ruleId}:${input.eventType}:${input.occurredAtMs}` + return makeCloudEvent({ + signal: { + sourceKind: "alert.lifecycle", + source: `urn:maple:alert-rule:${input.ruleId}`, + tenantId: input.tenantId, + occurrenceId, + identityQuality: "source", + occurredAt, + observedAt: occurredAt, + subject: + input.incidentId === null + ? `alert-rules/${input.ruleId}` + : `alert-incidents/${input.incidentId}`, + fields: new Map(), + data: {}, + }, + projection: { + id: "alert-lifecycle", + revision: 1, + enabled: true, + tenantId: input.tenantId, + sourceKind: "alert.lifecycle", + selector: { + op: "exists", + field: { namespace: "signal", key: "event_type", type: "string" }, + }, + projector: { id: "alert.lifecycle", version: 1, config: {} }, + activeFrom: occurredAt, + }, + projectorId: "alert.lifecycle", + projectorVersion: 1, + outputType: `dev.maple.alert.lifecycle.${input.eventType}.v1`, + dataSchema: "urn:maple:event-schema:alert-lifecycle:v1", + data: { + eventType: input.eventType, + incidentId: input.incidentId, + incidentStatus: input.incidentStatus, + rule: { + id: input.ruleId, + name: input.ruleName, + signalType: input.signalType, + severity: input.severity, + groupKey: input.groupKey, + comparator: input.comparator, + threshold: input.threshold, + thresholdUpper: input.thresholdUpper, + windowMinutes: input.windowMinutes, + }, + observed: { value: input.value, sampleCount: input.sampleCount }, + }, + }) +} + +export interface AlertLifecycleInput { + readonly policy: AlertLifecyclePolicy + readonly evaluation: AlertEvaluation + readonly state: AlertLifecycleState | null + readonly openIncident: AlertLifecycleIncident | null + readonly nowMs: number + /** Most recent notification for a resolved incident with the same rule and group. */ + readonly previousNotificationAtMs?: number | null + /** Set only after the host's telemetry-query adapter proves data is still arriving. */ + readonly allowNoDataResolution?: boolean +} + +export interface AlertLifecyclePlan { + readonly state: AlertLifecycleState + readonly transition: AlertIncidentTransition + readonly eventType: AlertEventType | null + readonly notificationSuppression: AlertNotificationSuppression + readonly hold: AlertLifecycleHold + /** Notification anchor to copy to a newly opened, flap-suppressed incident. */ + readonly inheritedNotificationAtMs: number | null + /** Whether the host must advance lastNotifiedAt before queueing the event. */ + readonly advanceNotificationAnchor: boolean +} + +export interface AlertDeliveryRetryPolicy { + readonly maxAttempts: number + readonly baseDelayMs: number + readonly maxDelayMs: number +} + +export const DEFAULT_ALERT_DELIVERY_RETRY_POLICY: AlertDeliveryRetryPolicy = { + maxAttempts: 5, + baseDelayMs: 60_000, + maxDelayMs: 15 * 60_000, +} + +/** Stable idempotency key shared by every alert delivery adapter. */ +export const makeAlertDeliveryKey = ( + incidentId: string, + destinationId: string, + eventType: AlertEventType, + scheduledAtMs: number, +): string => [incidentId, destinationId, eventType, scheduledAtMs].join(":") + +export const canRetryAlertDelivery = ( + attemptNumber: number, + retryable: boolean, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): boolean => retryable && attemptNumber < policy.maxAttempts + +/** Exponential retry delay; the host supplies jitter from its own random source. */ +export const alertDeliveryRetryDelayMs = ( + attemptNumber: number, + jitterMs: number, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): number => { + const exponent = Math.max(0, attemptNumber - 1) + const base = Math.min(policy.baseDelayMs * Math.pow(2, exponent), policy.maxDelayMs) + return base + Math.max(0, Math.floor(jitterMs)) +} + +const noTransition = (state: AlertLifecycleState, hold: AlertLifecycleHold = null): AlertLifecyclePlan => ({ + state, + transition: "none", + eventType: null, + notificationSuppression: null, + hold, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, +}) + +/** + * Decide the next alert state and lifecycle intent without performing I/O. + * + * The caller owns persistence, incident identifiers, delivery, telemetry + * liveness checks, and time. This makes the same lifecycle semantics usable by + * the hosted PostgreSQL/Tinybird adapter and a future Maple Local adapter. + */ +export const planAlertLifecycle = (input: AlertLifecycleInput): AlertLifecyclePlan => { + const { evaluation, policy, openIncident, nowMs } = input + const previous = input.state ?? { consecutiveBreaches: 0, consecutiveHealthy: 0 } + + if (evaluation.status === "skipped") return noTransition(previous) + + const state: AlertLifecycleState = { + consecutiveBreaches: + evaluation.status === "breached" + ? Math.min(previous.consecutiveBreaches + 1, policy.consecutiveBreachesRequired) + : 0, + consecutiveHealthy: + evaluation.status === "healthy" + ? Math.min(previous.consecutiveHealthy + 1, policy.consecutiveHealthyRequired) + : 0, + } + + if ( + evaluation.status === "breached" && + openIncident == null && + state.consecutiveBreaches >= policy.consecutiveBreachesRequired + ) { + const previousNotificationAtMs = input.previousNotificationAtMs ?? null + const flapSuppressed = + previousNotificationAtMs != null && + previousNotificationAtMs >= nowMs - policy.renotifyIntervalMinutes * 60_000 + return { + state, + transition: "opened", + eventType: flapSuppressed ? null : "trigger", + notificationSuppression: flapSuppressed ? "flapping" : null, + hold: null, + inheritedNotificationAtMs: flapSuppressed ? previousNotificationAtMs : null, + advanceNotificationAnchor: false, + } + } + + if (evaluation.status === "breached" && openIncident != null) { + const renotifyDueAt = + (openIncident.lastNotifiedAtMs ?? openIncident.firstTriggeredAtMs) + + policy.renotifyIntervalMinutes * 60_000 + const renotifyDue = renotifyDueAt <= nowMs + return { + state, + transition: "continued", + eventType: renotifyDue ? "renotify" : null, + notificationSuppression: null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: renotifyDue, + } + } + + if ( + evaluation.status === "healthy" && + openIncident != null && + state.consecutiveHealthy >= policy.consecutiveHealthyRequired + ) { + if (evaluation.derivedFromNoData && input.allowNoDataResolution !== true) { + return noTransition(state, "missing_telemetry") + } + + const flapResolutionSuppressed = + openIncident.lastDeliveredEventType == null && openIncident.lastNotifiedAtMs != null + return { + state, + transition: "resolved", + eventType: flapResolutionSuppressed ? null : "resolve", + notificationSuppression: flapResolutionSuppressed ? "flap_resolution" : null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, + } + } + + return noTransition(state) +} + +/** Preserve per-tenant order while preventing one tenant from monopolizing a tick. */ +export const interleaveAlertRulesByTenant = ( + rows: ReadonlyArray, + tenantIdOf: (row: T) => string, +): ReadonlyArray => { + const queues = new Map() + for (const row of rows) { + const tenantId = tenantIdOf(row) + const queue = queues.get(tenantId) + if (queue) queue.push(row) + else queues.set(tenantId, [row]) + } + + const fair: T[] = [] + let index = 0 + while (fair.length < rows.length) { + for (const queue of queues.values()) { + const row = queue[index] + if (row !== undefined) fair.push(row) + } + index += 1 + } + return fair +} diff --git a/packages/alerting-core/tsconfig.json b/packages/alerting-core/tsconfig.json new file mode 100644 index 000000000..3d83a7d0c --- /dev/null +++ b/packages/alerting-core/tsconfig.json @@ -0,0 +1,17 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + } +} diff --git a/packages/eventing-core/README.md b/packages/eventing-core/README.md new file mode 100644 index 000000000..9e5a883c3 --- /dev/null +++ b/packages/eventing-core/README.md @@ -0,0 +1,22 @@ +# `@maple/eventing-core` + +Host-neutral signal-to-event contracts and deterministic runtime semantics. + +The package owns typed signal values, bounded selectors, pure projector +registration, canonical event identity, and an immutable compiled projection +registry. It has no database, network, scheduler, or wall-clock dependency. A +host authenticates and normalizes source input, supplies durable projection and +outbox adapters, and decides when compiled registries become active. + +See [`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md) +for the architecture and acceptance contract. + +The versioned interoperability artifacts are generated under `schemas/`, with +valid comparison and identity vectors in `fixtures/v1.json`. Run `bun test` to +verify generated-schema drift, hostile selector bounds, typed comparison +semantics, deterministic event IDs, and projector isolation. + +The first host adapter is Maple Local in `apps/cli/src/server/eventing`. It uses +an authenticated configuration endpoint, a SQLite projection/outbox store, and +the pre-chDB OTLP seam. The package itself deliberately contains none of those +host decisions. diff --git a/packages/eventing-core/fixtures/v1.json b/packages/eventing-core/fixtures/v1.json new file mode 100644 index 000000000..9bd689b6d --- /dev/null +++ b/packages/eventing-core/fixtures/v1.json @@ -0,0 +1,191 @@ +{ + "version": 1, + "eventIdVectors": [ + { + "name": "tenant-scoped projected occurrence", + "input": { + "tenantId": "tenant-a", + "sourceKind": "otel.log", + "source": "urn:maple:source:otel:local", + "occurrenceId": "event-123", + "projectionId": "gitlab-issue-created", + "projectionRevision": 3 + }, + "output": "sha256:061c0b5d99b92ef65ab8813c6d84988e4b1582e705e0077c952e62a0e84b6b08" + } + ], + "stringLiteralByteVectors": [ + { + "name": "multibyte literal exactly at the UTF-8 byte limit", + "unit": "é", + "repeat": 2048, + "valid": true + }, + { + "name": "multibyte literal one code point beyond the UTF-8 byte limit", + "unit": "é", + "repeat": 2049, + "valid": false + } + ], + "predicateVectors": [ + { + "name": "int64 remains exact above JavaScript safe integer range", + "predicate": { + "op": "gt", + "field": { "namespace": "attribute", "key": "counter", "type": "int64" }, + "value": { "type": "int64", "value": "9007199254740992" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "counter", + "value": { "type": "int64", "value": "9007199254740993" } + } + ], + "matches": true + }, + { + "name": "timestamps compare as UTC instants", + "predicate": { + "op": "eq", + "field": { "namespace": "signal", "key": "occurred_at", "type": "timestamp" }, + "value": { "type": "timestamp", "value": "2026-08-07T19:42:00.123456789Z" } + }, + "fields": [ + { + "namespace": "signal", + "key": "occurred_at", + "value": { "type": "timestamp", "value": "2026-08-07T15:42:00.123456789-04:00" } + } + ], + "matches": true + }, + { + "name": "numeric strings do not coerce", + "predicate": { + "op": "gte", + "field": { "namespace": "attribute", "key": "attempt", "type": "int64" }, + "value": { "type": "int64", "value": "3" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "attempt", + "value": { "type": "string", "value": "12" } + } + ], + "matches": false, + "typeMismatches": ["attribute:attempt"] + }, + { + "name": "neq does not match a missing field", + "predicate": { + "op": "neq", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "value": { "type": "string", "value": "closed" } + }, + "fields": [], + "matches": false + }, + { + "name": "boolean composition and string containment", + "predicate": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "attribute", "key": "active", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + { + "op": "contains", + "field": { "namespace": "body", "key": "text", "type": "string" }, + "value": { "type": "string", "value": "issue created" } + } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "active", + "value": { "type": "boolean", "value": true } + }, + { + "namespace": "body", + "key": "text", + "value": { "type": "string", "value": "gitlab issue created successfully" } + } + ], + "matches": true + }, + { + "name": "float64 ordering is numeric", + "predicate": { + "op": "lt", + "field": { "namespace": "attribute", "key": "ratio", "type": "float64" }, + "value": { "type": "float64", "value": 10.25 } + }, + "fields": [ + { + "namespace": "attribute", + "key": "ratio", + "value": { "type": "float64", "value": 9.5 } + } + ], + "matches": true + }, + { + "name": "durations compare as exact nanoseconds", + "predicate": { + "op": "gte", + "field": { "namespace": "signal", "key": "duration", "type": "duration" }, + "value": { "type": "duration", "value": "1000000000" } + }, + "fields": [ + { + "namespace": "signal", + "key": "duration", + "value": { "type": "duration", "value": "1000000001" } + } + ], + "matches": true + }, + { + "name": "boolean equality has no string coercion", + "predicate": { + "op": "eq", + "field": { "namespace": "attribute", "key": "enabled", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + "fields": [ + { + "namespace": "attribute", + "key": "enabled", + "value": { "type": "string", "value": "true" } + } + ], + "matches": false, + "typeMismatches": ["attribute:enabled"] + }, + { + "name": "string membership is exact and case-sensitive", + "predicate": { + "op": "in", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "values": [ + { "type": "string", "value": "opened" }, + { "type": "string", "value": "closed" } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "state", + "value": { "type": "string", "value": "Closed" } + } + ], + "matches": false + } + ] +} diff --git a/packages/eventing-core/package.json b/packages/eventing-core/package.json new file mode 100644 index 000000000..3dd387ac9 --- /dev/null +++ b/packages/eventing-core/package.json @@ -0,0 +1,24 @@ +{ + "name": "@maple/eventing-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "schemas": "bun run scripts/generate-schemas.ts", + "schemas:check": "bun run scripts/generate-schemas.ts --check", + "test": "bun run schemas:check && vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "effect": "catalog:effect" + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/eventing-core/schemas/cloud-event.v1.schema.json b/packages/eventing-core/schemas/cloud-event.v1.schema.json new file mode 100644 index 000000000..323056917 --- /dev/null +++ b/packages/eventing-core/schemas/cloud-event.v1.schema.json @@ -0,0 +1,180 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:cloud-event:v1", + "$ref": "#/$defs/MapleCloudEvent", + "$defs": { + "MapleCloudEvent": { + "type": "object", + "properties": { + "specversion": { + "type": "string", + "enum": ["1.0"] + }, + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "subject": { + "type": "string" + }, + "time": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + }, + "datacontenttype": { + "type": "string", + "enum": ["application/json"] + }, + "dataschema": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "tenantid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionrevision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "projectorid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectorversion": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "sourceoccurrenceid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "sourceidentityquality": { + "type": "string", + "enum": ["source", "derived", "none"] + }, + "data": {} + }, + "required": [ + "specversion", + "id", + "source", + "type", + "time", + "datacontenttype", + "dataschema", + "tenantid", + "projectionid", + "projectionrevision", + "projectorid", + "projectorversion", + "data" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-deployment-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-deployment-lifecycle.v1.schema.json new file mode 100644 index 000000000..b6b3a048d --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-deployment-lifecycle.v1.schema.json @@ -0,0 +1,190 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-deployment-lifecycle:v1", + "$ref": "#/$defs/GitLabDeploymentLifecycleDataV1", + "$defs": { + "GitLabDeploymentLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "deployment": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "environment": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "status": { + "type": "string", + "enum": ["running", "success", "failed", "canceled", "blocked", "manual"] + }, + "revision": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + } + }, + "required": ["id", "environment", "status", "revision", "url"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "deployment", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-issue-comment.v1.schema.json b/packages/eventing-core/schemas/gitlab-issue-comment.v1.schema.json new file mode 100644 index 000000000..ccfd79aad --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-issue-comment.v1.schema.json @@ -0,0 +1,280 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-issue-comment:v1", + "$ref": "#/$defs/GitLabIssueCommentDataV1", + "$defs": { + "GitLabIssueCommentDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "issue": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "state": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + } + ] + }, + "allOf": [ + { + "maxItems": 50 + } + ] + } + }, + "required": ["iid", "type"], + "additionalProperties": false + }, + "comment": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "kind": { + "type": "string", + "enum": ["comment", "review"] + }, + "excerpt": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 1024 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?(?:#[^?#\\s]+)?$" + } + ] + }, + "system": { + "type": "boolean" + } + }, + "required": ["id", "excerpt", "url"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "issue", "comment", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-issue-created.v1.schema.json b/packages/eventing-core/schemas/gitlab-issue-created.v1.schema.json new file mode 100644 index 000000000..08f996297 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-issue-created.v1.schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-issue-created:v1", + "$ref": "#/$defs/GitLabIssueCreatedDataV1", + "$defs": { + "GitLabIssueCreatedDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "path": { + "type": "string" + } + }, + "required": ["path"], + "additionalProperties": false + }, + "issue": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "iid": { + "type": "string" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": ["iid"], + "additionalProperties": false + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "additionalProperties": false + }, + "serviceName": { + "type": "string" + }, + "body": {} + }, + "required": ["project", "issue"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-issue-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-issue-lifecycle.v1.schema.json new file mode 100644 index 000000000..9aebaee8b --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-issue-lifecycle.v1.schema.json @@ -0,0 +1,234 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-issue-lifecycle:v1", + "$ref": "#/$defs/GitLabIssueLifecycleDataV1", + "$defs": { + "GitLabIssueLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "issue": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "state": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + } + ] + }, + "allOf": [ + { + "maxItems": 50 + } + ] + }, + "action": { + "type": "string", + "enum": ["open", "update", "close", "reopen"] + } + }, + "required": ["iid", "type", "action"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "issue", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-job-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-job-lifecycle.v1.schema.json new file mode 100644 index 000000000..21f1edcfa --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-job-lifecycle.v1.schema.json @@ -0,0 +1,249 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-job-lifecycle:v1", + "$ref": "#/$defs/GitLabJobLifecycleDataV1", + "$defs": { + "GitLabJobLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "job": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "status": { + "type": "string", + "enum": [ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled" + ] + }, + "stage": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "pipelineId": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "ref": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "revision": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "durationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "allowFailure": { + "type": "boolean" + } + }, + "required": ["id", "name", "status"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "job", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-merge-request-comment.v1.schema.json b/packages/eventing-core/schemas/gitlab-merge-request-comment.v1.schema.json new file mode 100644 index 000000000..390003681 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-merge-request-comment.v1.schema.json @@ -0,0 +1,272 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-merge-request-comment:v1", + "$ref": "#/$defs/GitLabMergeRequestCommentDataV1", + "$defs": { + "GitLabMergeRequestCommentDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "mergeRequest": { + "type": "object", + "properties": { + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "sourceBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "targetBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "commit": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "reviewState": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + } + }, + "required": ["iid", "title", "url"], + "additionalProperties": false + }, + "comment": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "kind": { + "type": "string", + "enum": ["comment", "review"] + }, + "excerpt": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 1024 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?(?:#[^?#\\s]+)?$" + } + ] + }, + "system": { + "type": "boolean" + } + }, + "required": ["id", "excerpt", "url"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "mergeRequest", "comment", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-merge-request-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-merge-request-lifecycle.v1.schema.json new file mode 100644 index 000000000..523482edb --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-merge-request-lifecycle.v1.schema.json @@ -0,0 +1,226 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", + "$ref": "#/$defs/GitLabMergeRequestLifecycleDataV1", + "$defs": { + "GitLabMergeRequestLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "mergeRequest": { + "type": "object", + "properties": { + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "title": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "sourceBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "targetBranch": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "commit": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "reviewState": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "action": { + "type": "string", + "enum": ["open", "update", "close", "reopen", "merge", "review"] + } + }, + "required": ["iid", "title", "url", "action"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "mergeRequest", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-pipeline-completed.v1.schema.json b/packages/eventing-core/schemas/gitlab-pipeline-completed.v1.schema.json new file mode 100644 index 000000000..c429067d2 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-pipeline-completed.v1.schema.json @@ -0,0 +1,518 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-pipeline-completed:v1", + "$ref": "#/$defs/GitLabPipelineCompletedDataV1", + "$defs": { + "GitLabPipelineCompletedDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "pipeline": { + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "mergeRequestIid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "detailedStatus": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "ref": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "sha": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "durationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "queuedDurationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "stageCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "status": { + "type": "string", + "enum": ["failed"] + }, + "failedJobCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "failedJobsTruncated": { + "type": "boolean" + }, + "failedJobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "stage": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "status": { + "type": "string", + "enum": ["failed"] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + } + }, + "required": ["id", "name", "status"], + "additionalProperties": false + }, + "allOf": [ + { + "maxItems": 20 + } + ] + } + }, + "required": [ + "id", + "url", + "status", + "failedJobCount", + "failedJobsTruncated", + "failedJobs" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "iid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "mergeRequestIid": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "detailedStatus": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "ref": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "sha": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "durationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "queuedDurationMs": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "stageCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + }, + "status": { + "type": "string", + "enum": ["success", "canceled", "skipped"] + } + }, + "required": ["id", "url", "status"], + "additionalProperties": false + } + ] + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "pipeline", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-project-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-project-lifecycle.v1.schema.json new file mode 100644 index 000000000..548b7097f --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-project-lifecycle.v1.schema.json @@ -0,0 +1,142 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-project-lifecycle:v1", + "$ref": "#/$defs/GitLabProjectLifecycleDataV1", + "$defs": { + "GitLabProjectLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "action": { + "type": "string", + "enum": [ + "created", + "destroyed", + "renamed", + "transferred", + "updated", + "archived", + "unarchived", + "deletion_requested" + ] + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "action", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-ref-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-ref-lifecycle.v1.schema.json new file mode 100644 index 000000000..2587826d7 --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-ref-lifecycle.v1.schema.json @@ -0,0 +1,208 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-ref-lifecycle:v1", + "$ref": "#/$defs/GitLabRefLifecycleDataV1", + "$defs": { + "GitLabRefLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "ref": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["branch", "tag"] + }, + "action": { + "type": "string", + "enum": ["create", "update", "delete"] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "before": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "after": { + "type": "string", + "allOf": [ + { + "minLength": 6 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[0-9a-f]+$" + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + }, + "commitCount": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "action", "name", "before", "after"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "ref", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/gitlab-release-lifecycle.v1.schema.json b/packages/eventing-core/schemas/gitlab-release-lifecycle.v1.schema.json new file mode 100644 index 000000000..42a85cfeb --- /dev/null +++ b/packages/eventing-core/schemas/gitlab-release-lifecycle.v1.schema.json @@ -0,0 +1,187 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:event-schema:gitlab-release-lifecycle:v1", + "$ref": "#/$defs/GitLabReleaseLifecycleDataV1", + "$defs": { + "GitLabReleaseLifecycleDataV1": { + "type": "object", + "properties": { + "project": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "path": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "oldPath": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["id", "path"], + "additionalProperties": false + }, + "release": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "tag": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "action": { + "type": "string", + "enum": ["create", "update", "delete"] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "url": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 2048 + }, + { + "pattern": "^https?:\\/\\/[^/?#@\\s]+(?:\\/[^?#\\s]*)?$" + } + ] + } + }, + "required": ["tag", "action"], + "additionalProperties": false + }, + "sourceEvent": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 64 + }, + { + "pattern": "^[a-z][a-z0-9_]*$" + } + ] + }, + "actor": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^[1-9][0-9]*$" + } + ] + }, + "name": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "username": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "additionalProperties": false + }, + "result": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + }, + "serviceName": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 4096 + } + ] + } + }, + "required": ["project", "release", "sourceEvent"], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-projection.v1.schema.json b/packages/eventing-core/schemas/signal-projection.v1.schema.json new file mode 100644 index 000000000..0523586da --- /dev/null +++ b/packages/eventing-core/schemas/signal-projection.v1.schema.json @@ -0,0 +1,380 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-projection:v1", + "$ref": "#/$defs/SignalProjectionSpec", + "$defs": { + "SignalFieldRef": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "enum": ["signal", "resource", "scope", "attribute", "body"] + }, + "key": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 512 + } + ] + }, + "type": { + "type": "string", + "enum": ["string", "boolean", "int64", "float64", "timestamp", "duration"] + } + }, + "required": ["namespace", "key", "type"], + "additionalProperties": false + }, + "SignalLiteral": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string", + "description": "Predicate string literal limited to 4096 UTF-8 bytes; JSON Schema cannot express this byte-count constraint" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + }, + "SignalPredicate": { + "anyOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["all"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 64 + } + ] + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["any"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 64 + } + ] + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["not"] + }, + "clause": { + "$ref": "#/$defs/SignalPredicate" + } + }, + "required": ["op", "clause"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["exists"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + } + }, + "required": ["op", "field"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["eq", "neq", "gt", "gte", "lt", "lte", "contains"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "value": { + "$ref": "#/$defs/SignalLiteral" + } + }, + "required": ["op", "field", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["in"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalLiteral" + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 100 + } + ] + } + }, + "required": ["op", "field", "values"], + "additionalProperties": false + } + ] + }, + "SignalProjectionSpec": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "revision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "enabled": { + "type": "boolean" + }, + "tenantId": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "sourceKind": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "selector": { + "$ref": "#/$defs/SignalPredicate" + }, + "projector": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "version": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "config": {} + }, + "required": ["id", "version", "config"], + "additionalProperties": false + }, + "activeFrom": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": [ + "id", + "revision", + "enabled", + "tenantId", + "sourceKind", + "selector", + "projector", + "activeFrom" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-scalar.v1.schema.json b/packages/eventing-core/schemas/signal-scalar.v1.schema.json new file mode 100644 index 000000000..d83d0d9a2 --- /dev/null +++ b/packages/eventing-core/schemas/signal-scalar.v1.schema.json @@ -0,0 +1,116 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-scalar:v1", + "$ref": "#/$defs/SignalScalar", + "$defs": { + "SignalScalar": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + } + } +} diff --git a/packages/eventing-core/scripts/generate-schemas.ts b/packages/eventing-core/scripts/generate-schemas.ts new file mode 100644 index 000000000..c507e05a9 --- /dev/null +++ b/packages/eventing-core/scripts/generate-schemas.ts @@ -0,0 +1,67 @@ +import { execFileSync } from "node:child_process" +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { dirname, resolve } from "node:path" +import { Schema } from "effect" +import { MapleCloudEventSchema, SignalProjectionSpecSchema, SignalScalarSchema } from "../src/model" +import { GITLAB_DATA_SCHEMAS } from "../src/gitlab" + +const root = resolve(import.meta.dirname, "..") +const check = process.argv.includes("--check") + +const documents = [ + { + path: "schemas/signal-scalar.v1.schema.json", + id: "urn:maple:eventing:schema:signal-scalar:v1", + schema: SignalScalarSchema, + }, + { + path: "schemas/signal-projection.v1.schema.json", + id: "urn:maple:eventing:schema:signal-projection:v1", + schema: SignalProjectionSpecSchema, + }, + { + path: "schemas/cloud-event.v1.schema.json", + id: "urn:maple:eventing:schema:cloud-event:v1", + schema: MapleCloudEventSchema, + }, + ...GITLAB_DATA_SCHEMAS.map(([id, schema]) => ({ + path: `schemas/${id.slice("urn:maple:event-schema:".length).replace(":v1", ".v1")}.schema.json`, + id, + schema, + })), +] as const + +let stale = false +for (const entry of documents) { + const document = Schema.toJsonSchemaDocument(entry.schema) + const unformatted = `${JSON.stringify( + { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: entry.id, + ...document.schema, + ...(Object.keys(document.definitions).length === 0 ? {} : { $defs: document.definitions }), + }, + null, + "\t", + )}\n` + const serialized = execFileSync( + resolve(root, "../../node_modules/.bin/oxfmt"), + ["--stdin-filepath", entry.path], + { + input: unformatted, + encoding: "utf8", + }, + ) + const path = resolve(root, entry.path) + if (check) { + if (!existsSync(path) || readFileSync(path, "utf8") !== serialized) { + console.error(`${entry.path} is stale; run bun run schemas`) + stale = true + } + } else { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, serialized) + } +} + +if (stale) process.exitCode = 1 diff --git a/packages/eventing-core/src/event.ts b/packages/eventing-core/src/event.ts new file mode 100644 index 000000000..f21e92b43 --- /dev/null +++ b/packages/eventing-core/src/event.ts @@ -0,0 +1,149 @@ +import { createHash } from "node:crypto" +import { Schema } from "effect" +import { + MapleCloudEventSchema, + type JsonValue, + type MapleCloudEvent, + type NormalizedSignal, + type SignalProjectionSpec, +} from "./model" +import { timestampToEpochNanos } from "./predicate" + +export const MAX_CLOUD_EVENT_BYTES = 256 * 1024 + +export interface EventIdentityInput { + readonly tenantId: string + readonly sourceKind: string + readonly source: string + readonly occurrenceId: string + readonly projectionId: string + readonly projectionRevision: number +} + +const updateLengthDelimited = (hash: ReturnType, value: string): void => { + const encoded = Buffer.from(value, "utf8") + const length = Buffer.allocUnsafe(4) + length.writeUInt32BE(encoded.byteLength) + hash.update(length) + hash.update(encoded) +} + +/** Canonical v1 identity shared by every host implementation. */ +export const makeEventId = (input: EventIdentityInput): string => { + const hash = createHash("sha256") + for (const field of [ + "maple-event-v1", + input.tenantId, + input.sourceKind, + input.source, + input.occurrenceId, + input.projectionId, + String(input.projectionRevision), + ]) + updateLengthDelimited(hash, field) + return `sha256:${hash.digest("hex")}` +} + +export const isJsonValue = (value: unknown, seen: Set = new Set()): value is JsonValue => { + if (value === null || typeof value === "string" || typeof value === "boolean") return true + if (typeof value === "number") return Number.isFinite(value) + if (typeof value !== "object") return false + if (seen.has(value)) return false + seen.add(value) + try { + if (Array.isArray(value)) return value.every((item) => isJsonValue(item, seen)) + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return false + return Object.values(value).every((item) => isJsonValue(item, seen)) + } finally { + // Track the active recursion path. Repeated references serialize as a + // JSON tree and are not themselves cycles. + seen.delete(value) + } +} + +const canonicalizeJson = (value: JsonValue): JsonValue => { + if (value === null || typeof value !== "object") return value + if (Array.isArray(value)) return value.map(canonicalizeJson) + const record = value as { readonly [key: string]: JsonValue } + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalizeJson(record[key]!)]), + ) +} + +/** Stable JSON encoding for outbox collision checks and cross-host fixtures. */ +export const canonicalJson = (value: JsonValue): string => { + if (!isJsonValue(value)) throw new Error("value must be finite acyclic JSON") + return JSON.stringify(canonicalizeJson(value)) +} + +export interface ValidatedMapleCloudEvent { + readonly event: MapleCloudEvent + readonly canonicalJson: string + readonly byteLength: number +} + +/** Validate the complete persisted envelope, including its canonical byte budget. */ +export const validateMapleCloudEvent = (candidate: unknown): ValidatedMapleCloudEvent => { + const event = Schema.decodeUnknownSync(MapleCloudEventSchema)(candidate) + if (!isJsonValue(event as unknown)) throw new Error("CloudEvent must be finite JSON") + const eventJson = canonicalJson(event as unknown as JsonValue) + const byteLength = Buffer.byteLength(eventJson, "utf8") + if (byteLength > MAX_CLOUD_EVENT_BYTES) + throw new Error(`CloudEvent exceeds ${MAX_CLOUD_EVENT_BYTES} UTF-8 bytes`) + return { event: event as MapleCloudEvent, canonicalJson: eventJson, byteLength } +} + +export const makeCloudEvent = (input: { + readonly signal: NormalizedSignal + readonly projection: SignalProjectionSpec + readonly projectorId: string + readonly projectorVersion: number + readonly outputType: string + readonly dataSchema: string + readonly subject?: string | null + readonly time?: string + readonly data: JsonValue +}): MapleCloudEvent => { + if ( + input.signal.occurrenceId === null || + input.signal.occurrenceId.trim().length === 0 || + input.signal.identityQuality === "none" + ) + throw new Error("durable event projection requires stable or derived occurrence identity") + if (!isJsonValue(input.data)) throw new Error("projected event data must be finite JSON") + if (input.outputType.trim().length === 0) throw new Error("projected event type must not be empty") + if (input.dataSchema.trim().length === 0) throw new Error("projected event data schema must not be empty") + if (input.signal.source.trim().length === 0) throw new Error("signal source must not be empty") + + const subject = input.subject ?? input.signal.subject + const time = input.time ?? input.signal.occurredAt + if (timestampToEpochNanos(time) === null) throw new Error("projected event time must be a valid instant") + return validateMapleCloudEvent({ + specversion: "1.0", + id: makeEventId({ + tenantId: input.signal.tenantId, + sourceKind: input.signal.sourceKind, + source: input.signal.source, + occurrenceId: input.signal.occurrenceId, + projectionId: input.projection.id, + projectionRevision: input.projection.revision, + }), + source: input.signal.source, + type: input.outputType, + ...(subject == null ? {} : { subject }), + time, + datacontenttype: "application/json", + dataschema: input.dataSchema, + tenantid: input.signal.tenantId, + projectionid: input.projection.id, + projectionrevision: input.projection.revision, + projectorid: input.projectorId, + projectorversion: input.projectorVersion, + sourceoccurrenceid: input.signal.occurrenceId, + sourceidentityquality: input.signal.identityQuality, + data: input.data, + }).event +} diff --git a/packages/eventing-core/src/gitlab.ts b/packages/eventing-core/src/gitlab.ts new file mode 100644 index 000000000..5063bfc18 --- /dev/null +++ b/packages/eventing-core/src/gitlab.ts @@ -0,0 +1,297 @@ +import { Schema } from "effect" + +const Text = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(4 * 1024)) +const CanonicalUrl = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(2 * 1024), + Schema.isPattern(/^https?:\/\/[^/?#@\s]+(?:\/[^?#\s]*)?$/), +) +const CommentUrl = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(2 * 1024), + Schema.isPattern(/^https?:\/\/[^/?#@\s]+(?:\/[^?#\s]*)?(?:#[^?#\s]+)?$/), +) +const PositiveInt64 = Schema.String.check(Schema.isMaxLength(20), Schema.isPattern(/^[1-9][0-9]*$/)) +const NonNegativeInt64 = Schema.String.check(Schema.isMaxLength(20), Schema.isPattern(/^(?:0|[1-9][0-9]*)$/)) +const Revision = Schema.String.check( + Schema.isMinLength(6), + Schema.isMaxLength(64), + Schema.isPattern(/^[0-9a-f]+$/i), +) +const Token = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(64), + Schema.isPattern(/^[a-z][a-z0-9_]*$/), +) +const CommentExcerpt = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(1024)) + +const Project = Schema.Struct({ + id: PositiveInt64, + path: Text, + oldPath: Schema.optionalKey(Text), +}) + +const Actor = Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + name: Schema.optionalKey(Text), + username: Schema.optionalKey(Text), +}) + +const Common = { + sourceEvent: Token, + actor: Schema.optionalKey(Actor), + result: Schema.optionalKey(Text), + serviceName: Schema.optionalKey(Text), +} as const + +const Issue = Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + iid: PositiveInt64, + type: Token, + title: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + state: Schema.optionalKey(Token), + labels: Schema.optionalKey( + Schema.Array(Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256))).check( + Schema.isMaxLength(50), + ), + ), +}) + +const MergeRequest = Schema.Struct({ + iid: PositiveInt64, + title: Text, + url: CanonicalUrl, + sourceBranch: Schema.optionalKey(Text), + targetBranch: Schema.optionalKey(Text), + commit: Schema.optionalKey(Revision), + reviewState: Schema.optionalKey(Token), +}) + +const Comment = Schema.Struct({ + id: PositiveInt64, + kind: Schema.optionalKey(Schema.Literals(["comment", "review"])), + excerpt: CommentExcerpt, + url: CommentUrl, + system: Schema.optionalKey(Schema.Boolean), +}) + +export const GitLabIssueCreatedDataSchema = Schema.Struct({ + project: Schema.Struct({ id: Schema.optionalKey(Schema.String), path: Schema.String }), + issue: Schema.Struct({ + id: Schema.optionalKey(Schema.String), + iid: Schema.String, + title: Schema.optionalKey(Schema.String), + url: Schema.optionalKey(Schema.String), + }), + actor: Schema.optionalKey( + Schema.Struct({ id: Schema.optionalKey(Schema.String), username: Schema.optionalKey(Schema.String) }), + ), + serviceName: Schema.optionalKey(Schema.String), + body: Schema.optionalKey(Schema.Unknown), +}).annotate({ identifier: "GitLabIssueCreatedDataV1" }) + +export const GitLabProjectLifecycleDataSchema = Schema.Struct({ + project: Project, + action: Schema.Literals([ + "created", + "destroyed", + "renamed", + "transferred", + "updated", + "archived", + "unarchived", + "deletion_requested", + ]), + ...Common, +}).annotate({ identifier: "GitLabProjectLifecycleDataV1" }) + +export const GitLabIssueLifecycleDataSchema = Schema.Struct({ + project: Project, + issue: Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + iid: PositiveInt64, + type: Token, + title: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + state: Schema.optionalKey(Token), + labels: Schema.optionalKey( + Schema.Array(Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256))).check( + Schema.isMaxLength(50), + ), + ), + action: Schema.Literals(["open", "update", "close", "reopen"]), + }), + ...Common, +}).annotate({ identifier: "GitLabIssueLifecycleDataV1" }) + +export const GitLabIssueCommentDataSchema = Schema.Struct({ + project: Project, + issue: Issue, + comment: Comment, + ...Common, +}).annotate({ identifier: "GitLabIssueCommentDataV1" }) + +export const GitLabMergeRequestLifecycleDataSchema = Schema.Struct({ + project: Project, + mergeRequest: Schema.Struct({ + iid: PositiveInt64, + title: Text, + url: CanonicalUrl, + sourceBranch: Schema.optionalKey(Text), + targetBranch: Schema.optionalKey(Text), + commit: Schema.optionalKey(Revision), + reviewState: Schema.optionalKey(Token), + action: Schema.Literals(["open", "update", "close", "reopen", "merge", "review"]), + }), + ...Common, +}).annotate({ identifier: "GitLabMergeRequestLifecycleDataV1" }) + +export const GitLabMergeRequestCommentDataSchema = Schema.Struct({ + project: Project, + mergeRequest: MergeRequest, + comment: Comment, + ...Common, +}).annotate({ identifier: "GitLabMergeRequestCommentDataV1" }) + +const PipelineCommon = { + id: PositiveInt64, + iid: Schema.optionalKey(PositiveInt64), + mergeRequestIid: Schema.optionalKey(PositiveInt64), + name: Schema.optionalKey(Text), + source: Schema.optionalKey(Text), + detailedStatus: Schema.optionalKey(Text), + url: CanonicalUrl, + ref: Schema.optionalKey(Text), + sha: Schema.optionalKey(Revision), + durationMs: Schema.optionalKey(NonNegativeInt64), + queuedDurationMs: Schema.optionalKey(NonNegativeInt64), + stageCount: Schema.optionalKey(NonNegativeInt64), +} as const + +const FailedJob = Schema.Struct({ + id: PositiveInt64, + name: Text, + stage: Schema.optionalKey(Text), + status: Schema.Literal("failed"), + url: Schema.optionalKey(CanonicalUrl), +}) + +const Pipeline = Schema.Union([ + Schema.Struct({ + ...PipelineCommon, + status: Schema.Literal("failed"), + failedJobCount: NonNegativeInt64, + failedJobsTruncated: Schema.Boolean, + failedJobs: Schema.Array(FailedJob).check(Schema.isMaxLength(20)), + }), + Schema.Struct({ + ...PipelineCommon, + status: Schema.Literals(["success", "canceled", "skipped"]), + }), +]) + +export const GitLabPipelineCompletedDataSchema = Schema.Struct({ + project: Project, + pipeline: Pipeline, + ...Common, +}).annotate({ identifier: "GitLabPipelineCompletedDataV1" }) + +export const GitLabDeploymentLifecycleDataSchema = Schema.Struct({ + project: Project, + deployment: Schema.Struct({ + id: PositiveInt64, + environment: Text, + status: Schema.Literals(["running", "success", "failed", "canceled", "blocked", "manual"]), + revision: Revision, + url: CanonicalUrl, + }), + ...Common, +}).annotate({ identifier: "GitLabDeploymentLifecycleDataV1" }) + +export const GitLabJobLifecycleDataSchema = Schema.Struct({ + project: Project, + job: Schema.Struct({ + id: PositiveInt64, + name: Text, + status: Schema.Literals([ + "created", + "pending", + "preparing", + "waiting_for_resource", + "running", + "success", + "failed", + "canceled", + "skipped", + "manual", + "scheduled", + ]), + stage: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + pipelineId: Schema.optionalKey(PositiveInt64), + ref: Schema.optionalKey(Text), + revision: Schema.optionalKey(Revision), + durationMs: Schema.optionalKey(NonNegativeInt64), + allowFailure: Schema.optionalKey(Schema.Boolean), + }), + ...Common, +}).annotate({ identifier: "GitLabJobLifecycleDataV1" }) + +export const GitLabRefLifecycleDataSchema = Schema.Struct({ + project: Project, + ref: Schema.Struct({ + type: Schema.Literals(["branch", "tag"]), + action: Schema.Literals(["create", "update", "delete"]), + name: Text, + before: Revision, + after: Revision, + url: Schema.optionalKey(CanonicalUrl), + commitCount: Schema.optionalKey(NonNegativeInt64), + }), + ...Common, +}).annotate({ identifier: "GitLabRefLifecycleDataV1" }) + +export const GitLabReleaseLifecycleDataSchema = Schema.Struct({ + project: Project, + release: Schema.Struct({ + id: Schema.optionalKey(PositiveInt64), + tag: Text, + action: Schema.Literals(["create", "update", "delete"]), + name: Schema.optionalKey(Text), + url: Schema.optionalKey(CanonicalUrl), + }), + ...Common, +}).annotate({ identifier: "GitLabReleaseLifecycleDataV1" }) + +export type GitLabIssueCreatedData = Schema.Schema.Type +export type GitLabProjectLifecycleData = Schema.Schema.Type +export type GitLabIssueLifecycleData = Schema.Schema.Type +export type GitLabIssueCommentData = Schema.Schema.Type +export type GitLabMergeRequestLifecycleData = Schema.Schema.Type +export type GitLabMergeRequestCommentData = Schema.Schema.Type +export type GitLabPipelineCompletedData = Schema.Schema.Type +export type GitLabDeploymentLifecycleData = Schema.Schema.Type +export type GitLabJobLifecycleData = Schema.Schema.Type +export type GitLabRefLifecycleData = Schema.Schema.Type +export type GitLabReleaseLifecycleData = Schema.Schema.Type + +export const GITLAB_DATA_SCHEMAS = [ + ["urn:maple:event-schema:gitlab-issue-created:v1", GitLabIssueCreatedDataSchema], + ["urn:maple:event-schema:gitlab-project-lifecycle:v1", GitLabProjectLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-issue-lifecycle:v1", GitLabIssueLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-issue-comment:v1", GitLabIssueCommentDataSchema], + ["urn:maple:event-schema:gitlab-merge-request-lifecycle:v1", GitLabMergeRequestLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-merge-request-comment:v1", GitLabMergeRequestCommentDataSchema], + ["urn:maple:event-schema:gitlab-pipeline-completed:v1", GitLabPipelineCompletedDataSchema], + ["urn:maple:event-schema:gitlab-deployment-lifecycle:v1", GitLabDeploymentLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-job-lifecycle:v1", GitLabJobLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-ref-lifecycle:v1", GitLabRefLifecycleDataSchema], + ["urn:maple:event-schema:gitlab-release-lifecycle:v1", GitLabReleaseLifecycleDataSchema], +] as const + +export const validateGitLabEventData = (dataSchema: string, candidate: unknown): unknown => { + const entry = GITLAB_DATA_SCHEMAS.find(([id]) => id === dataSchema) + if (entry === undefined) throw new Error(`unknown GitLab event data schema: ${dataSchema}`) + return Schema.decodeUnknownSync(entry[1] as Schema.Codec)(candidate) +} diff --git a/packages/eventing-core/src/index.ts b/packages/eventing-core/src/index.ts new file mode 100644 index 000000000..dffa0fdcd --- /dev/null +++ b/packages/eventing-core/src/index.ts @@ -0,0 +1,6 @@ +export * from "./event" +export * from "./gitlab" +export * from "./model" +export * from "./predicate" +export * from "./registry" +export * from "./source" diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts new file mode 100644 index 000000000..5bdca978f --- /dev/null +++ b/packages/eventing-core/src/model.ts @@ -0,0 +1,267 @@ +import { Schema } from "effect" + +export const MAX_PREDICATE_DEPTH = 8 +export const MAX_PREDICATE_NODES = 64 +export const MAX_IN_VALUES = 100 +export const MAX_STRING_LITERAL_BYTES = 4 * 1024 +export const MAX_DECIMAL_INT64_LENGTH = 20 + +const utf8Bytes = (value: string): number => new TextEncoder().encode(value).byteLength + +const NonEmptyIdentifier = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(256), + Schema.isTrimmed(), +) + +const DecimalInt64 = Schema.String.check( + Schema.isMaxLength(MAX_DECIMAL_INT64_LENGTH), + Schema.isPattern(/^-?(?:0|[1-9][0-9]*)$/), +) + +const Rfc3339Timestamp = Schema.String.check( + Schema.isPattern(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/), +) + +export const StringSignalScalar = Schema.Struct({ + type: Schema.Literal("string"), + value: Schema.String, +}) + +const StringLiteralValue = Schema.String.annotate({ + description: `Predicate string literal limited to ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes; JSON Schema cannot express this byte-count constraint`, +}).check( + Schema.makeFilter((value) => utf8Bytes(value) <= MAX_STRING_LITERAL_BYTES, { + expected: `a string no larger than ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`, + description: `Predicate string literal limited to ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`, + }), +) + +export const StringSignalLiteral = Schema.Struct({ + type: Schema.Literal("string"), + value: StringLiteralValue, +}) + +export const BooleanSignalScalar = Schema.Struct({ + type: Schema.Literal("boolean"), + value: Schema.Boolean, +}) + +export const Int64SignalScalar = Schema.Struct({ + type: Schema.Literal("int64"), + value: DecimalInt64, +}) + +export const Float64SignalScalar = Schema.Struct({ + type: Schema.Literal("float64"), + value: Schema.Finite, +}) + +export const TimestampSignalScalar = Schema.Struct({ + type: Schema.Literal("timestamp"), + value: Rfc3339Timestamp, +}) + +export const DurationSignalScalar = Schema.Struct({ + type: Schema.Literal("duration"), + value: DecimalInt64, +}) + +export const SignalScalarSchema = Schema.Union([ + StringSignalScalar, + BooleanSignalScalar, + Int64SignalScalar, + Float64SignalScalar, + TimestampSignalScalar, + DurationSignalScalar, +]).annotate({ identifier: "SignalScalar" }) +export type SignalScalar = Schema.Schema.Type +export type SignalScalarType = SignalScalar["type"] + +export const SignalLiteralSchema = Schema.Union([ + StringSignalLiteral, + BooleanSignalScalar, + Int64SignalScalar, + Float64SignalScalar, + TimestampSignalScalar, + DurationSignalScalar, +]).annotate({ identifier: "SignalLiteral" }) +export type SignalLiteral = Schema.Schema.Type + +export const FieldNamespaceSchema = Schema.Literals(["signal", "resource", "scope", "attribute", "body"]) +export type FieldNamespace = Schema.Schema.Type + +export const FieldRefSchema = Schema.Struct({ + namespace: FieldNamespaceSchema, + key: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(512)), + type: Schema.Literals(["string", "boolean", "int64", "float64", "timestamp", "duration"]), +}).annotate({ identifier: "SignalFieldRef" }) +export type FieldRef = Schema.Schema.Type + +export interface AllPredicate { + readonly op: "all" + readonly clauses: readonly SignalPredicate[] +} + +export interface AnyPredicate { + readonly op: "any" + readonly clauses: readonly SignalPredicate[] +} + +export interface NotPredicate { + readonly op: "not" + readonly clause: SignalPredicate +} + +export interface ExistsPredicate { + readonly op: "exists" + readonly field: FieldRef +} + +export interface ComparisonPredicate { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalLiteral +} + +export interface InPredicate { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalLiteral[] +} + +export type SignalPredicate = + | AllPredicate + | AnyPredicate + | NotPredicate + | ExistsPredicate + | ComparisonPredicate + | InPredicate + +export const SignalPredicateSchema: Schema.Codec = Schema.suspend( + (): Schema.Codec => + Schema.Union([ + Schema.Struct({ + op: Schema.Literal("all"), + clauses: Schema.Array(SignalPredicateSchema).check( + Schema.isMinLength(1), + Schema.isMaxLength(MAX_PREDICATE_NODES), + ), + }), + Schema.Struct({ + op: Schema.Literal("any"), + clauses: Schema.Array(SignalPredicateSchema).check( + Schema.isMinLength(1), + Schema.isMaxLength(MAX_PREDICATE_NODES), + ), + }), + Schema.Struct({ + op: Schema.Literal("not"), + clause: SignalPredicateSchema, + }), + Schema.Struct({ + op: Schema.Literal("exists"), + field: FieldRefSchema, + }), + Schema.Struct({ + op: Schema.Literals(["eq", "neq", "gt", "gte", "lt", "lte", "contains"]), + field: FieldRefSchema, + value: SignalLiteralSchema, + }), + Schema.Struct({ + op: Schema.Literal("in"), + field: FieldRefSchema, + values: Schema.Array(SignalLiteralSchema).check( + Schema.isMinLength(1), + Schema.isMaxLength(MAX_IN_VALUES), + ), + }), + ]) as Schema.Codec, +).annotate({ identifier: "SignalPredicate" }) + +export const ProjectorRefSchema = Schema.Struct({ + id: NonEmptyIdentifier, + version: Schema.Int.check(Schema.isGreaterThan(0)), + config: Schema.Unknown, +}) +export type ProjectorRef = Schema.Schema.Type + +export const SignalProjectionSpecSchema = Schema.Struct({ + id: NonEmptyIdentifier, + revision: Schema.Int.check(Schema.isGreaterThan(0)), + enabled: Schema.Boolean, + tenantId: NonEmptyIdentifier, + sourceKind: NonEmptyIdentifier, + selector: SignalPredicateSchema, + projector: ProjectorRefSchema, + activeFrom: Rfc3339Timestamp, +}).annotate({ identifier: "SignalProjectionSpec" }) +export type SignalProjectionSpec = Schema.Schema.Type + +export interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: TData +} + +export type JsonPrimitive = string | number | boolean | null +export type JsonValue = JsonPrimitive | { readonly [key: string]: JsonValue } | readonly JsonValue[] + +export interface ProjectedEventData { + readonly subject?: string | null + readonly time?: string + readonly data: JsonValue +} + +export interface MapleCloudEvent { + readonly specversion: "1.0" + readonly id: string + readonly source: string + readonly type: string + readonly subject?: string + readonly time: string + readonly datacontenttype: "application/json" + readonly dataschema: string + readonly tenantid: string + readonly projectionid: string + readonly projectionrevision: number + readonly projectorid: string + readonly projectorversion: number + readonly sourceoccurrenceid?: string + readonly sourceidentityquality?: "source" | "derived" | "none" + readonly data: JsonValue +} + +export const MapleCloudEventSchema = Schema.Struct({ + specversion: Schema.Literal("1.0"), + id: NonEmptyIdentifier, + source: NonEmptyIdentifier, + type: NonEmptyIdentifier, + subject: Schema.optionalKey(Schema.String), + time: Rfc3339Timestamp, + datacontenttype: Schema.Literal("application/json"), + dataschema: NonEmptyIdentifier, + tenantid: NonEmptyIdentifier, + projectionid: NonEmptyIdentifier, + projectionrevision: Schema.Int.check(Schema.isGreaterThan(0)), + projectorid: NonEmptyIdentifier, + projectorversion: Schema.Int.check(Schema.isGreaterThan(0)), + sourceoccurrenceid: Schema.optionalKey(NonEmptyIdentifier), + sourceidentityquality: Schema.optionalKey(Schema.Literals(["source", "derived", "none"])), + data: Schema.Unknown, +}).annotate({ identifier: "MapleCloudEvent" }) + +export const fieldKey = (field: Pick): string => + `${field.namespace}:${field.key}` + +export const defineSignalFields = ( + fields: ReadonlyArray<{ readonly field: FieldRef; readonly value: SignalScalar }>, +): ReadonlyMap => + new Map(fields.map(({ field, value }) => [fieldKey(field), value] as const)) diff --git a/packages/eventing-core/src/predicate.test.ts b/packages/eventing-core/src/predicate.test.ts new file mode 100644 index 000000000..12707dd4a --- /dev/null +++ b/packages/eventing-core/src/predicate.test.ts @@ -0,0 +1,224 @@ +import { readFileSync } from "node:fs" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" +import { + assertSignalProjectionInputBudget, + compileSignalPredicate, + defineSignalFields, + fieldKey, + makeEventId, + MAX_PREDICATE_DEPTH, + SignalLiteralSchema, + SignalPredicateSchema, + SignalScalarSchema, + timestampToEpochNanos, + validateSignalPredicate, + type EventIdentityInput, + type FieldNamespace, + type FieldRef, + type NormalizedSignal, +} from "./index" + +interface ConformanceFixture { + readonly eventIdVectors: ReadonlyArray<{ + readonly name: string + readonly input: EventIdentityInput + readonly output: string + }> + readonly stringLiteralByteVectors: ReadonlyArray<{ + readonly name: string + readonly unit: string + readonly repeat: number + readonly valid: boolean + }> + readonly predicateVectors: ReadonlyArray<{ + readonly name: string + readonly predicate: unknown + readonly fields: ReadonlyArray<{ + readonly namespace: FieldNamespace + readonly key: string + readonly value: unknown + }> + readonly matches: boolean + readonly typeMismatches?: readonly string[] + }> +} + +const fixture = JSON.parse( + readFileSync(new URL("../fixtures/v1.json", import.meta.url), "utf8"), +) as ConformanceFixture + +const signalFor = (fields: ConformanceFixture["predicateVectors"][number]["fields"]): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "occurrence-1", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00Z", + observedAt: "2026-08-07T19:42:01Z", + subject: null, + fields: defineSignalFields( + fields.map(({ namespace, key, value }) => ({ + field: { + namespace, + key, + type: Schema.decodeUnknownSync(SignalScalarSchema)(value).type, + }, + value: Schema.decodeUnknownSync(SignalScalarSchema)(value), + })), + ), + data: {}, +}) + +describe("cross-language conformance vectors", () => { + for (const vector of fixture.eventIdVectors) { + it(`event ID: ${vector.name}`, () => { + expect(makeEventId(vector.input)).toBe(vector.output) + }) + } + + for (const vector of fixture.stringLiteralByteVectors) { + it(`string literal bytes: ${vector.name}`, () => { + const candidate = { type: "string", value: vector.unit.repeat(vector.repeat) } + if (vector.valid) + expect(() => Schema.decodeUnknownSync(SignalLiteralSchema)(candidate)).not.toThrow() + else expect(() => Schema.decodeUnknownSync(SignalLiteralSchema)(candidate)).toThrow() + }) + } + + for (const vector of fixture.predicateVectors) { + it(`predicate: ${vector.name}`, () => { + const predicate = Schema.decodeUnknownSync(SignalPredicateSchema)(vector.predicate) + const result = compileSignalPredicate(predicate)(signalFor(vector.fields)) + expect(result.matches).toBe(vector.matches) + expect(result.typeMismatches.map(fieldKey)).toEqual(vector.typeMismatches ?? []) + }) + } +}) + +describe("selector validation", () => { + it("rejects wrong literal types and unsupported ordering", () => { + const field: FieldRef = { namespace: "attribute", key: "enabled", type: "boolean" } + expect( + validateSignalPredicate({ op: "gt", field, value: { type: "string", value: "true" } }), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ message: "gt is not supported for boolean" }), + expect.objectContaining({ message: "field and literal types must match" }), + ]), + ) + }) + + it("rejects empty combinators and excessive nesting", () => { + expect(validateSignalPredicate({ op: "all", clauses: [] })).toContainEqual({ + path: "selector.clauses", + message: "all requires at least one clause", + }) + + let nested = { + op: "exists" as const, + field: { namespace: "attribute" as const, key: "x", type: "string" as const }, + } + for (let i = 0; i < MAX_PREDICATE_DEPTH; i++) nested = { op: "not", clause: nested } as never + expect(validateSignalPredicate(nested)).toEqual( + expect.arrayContaining([expect.objectContaining({ message: `predicate depth exceeds 8` })]), + ) + }) + + it("rejects invalid calendar dates and int64 overflow", () => { + expect(timestampToEpochNanos("2026-02-31T00:00:00Z")).toBeNull() + expect( + validateSignalPredicate({ + op: "eq", + field: { namespace: "attribute", key: "n", type: "int64" }, + value: { type: "int64", value: "9223372036854775808" }, + }), + ).toContainEqual( + expect.objectContaining({ message: "int64 must be a signed 64-bit decimal integer" }), + ) + }) + + it("rejects hostile raw predicate topology before recursive schema decoding", () => { + let deeplyNested: unknown = { + op: "exists", + field: { namespace: "attribute", key: "x", type: "string" }, + } + for (let index = 0; index < MAX_PREDICATE_DEPTH; index++) + deeplyNested = { op: "not", clause: deeplyNested } + expect(() => assertSignalProjectionInputBudget({ selector: deeplyNested })).toThrow( + "predicate depth exceeds", + ) + + expect(() => + assertSignalProjectionInputBudget({ + selector: { + op: "all", + clauses: Array.from({ length: 65 }, () => ({ + op: "exists", + field: { namespace: "attribute", key: "x", type: "string" }, + })), + }, + }), + ).toThrow("clause list exceeds") + + expect(() => + assertSignalProjectionInputBudget({ + selector: { + op: "eq", + field: { namespace: "attribute", key: "n", type: "int64" }, + value: { type: "int64", value: "1".repeat(21) }, + }, + }), + ).toThrow("int64 literal exceeds") + }) +}) + +describe("total runtime behavior", () => { + it("accepts valid large source strings for exists and small contains literals", () => { + const largeValue = `${"a".repeat(5 * 1024)}needle` + const signal = signalFor([ + { + namespace: "attribute", + key: "large.description", + value: { type: "string", value: largeValue }, + }, + ]) + const field = { + namespace: "attribute" as const, + key: "large.description", + type: "string" as const, + } + + expect(compileSignalPredicate({ op: "exists", field })(signal).matches).toBe(true) + expect( + compileSignalPredicate({ + op: "contains", + field, + value: { type: "string", value: "needle" }, + })(signal).matches, + ).toBe(true) + }) + + it("treats malformed source scalars as mismatches rather than throwing", () => { + const field: FieldRef = { namespace: "attribute", key: "n", type: "int64" } + const evaluate = compileSignalPredicate({ + op: "gte", + field, + value: { type: "int64", value: "1" }, + }) + const signal = signalFor([]) + const fields = new Map(signal.fields) + fields.set(fieldKey(field), { type: "int64", value: "not-an-integer" }) + expect(evaluate({ ...signal, fields })).toMatchObject({ + matches: false, + typeMismatches: [field], + }) + }) + + it("distinguishes neq from not(eq) for a missing field", () => { + const field: FieldRef = { namespace: "attribute", key: "state", type: "string" } + const eq = { op: "eq" as const, field, value: { type: "string" as const, value: "closed" } } + expect(compileSignalPredicate({ ...eq, op: "neq" })(signalFor([])).matches).toBe(false) + expect(compileSignalPredicate({ op: "not", clause: eq })(signalFor([])).matches).toBe(true) + }) +}) diff --git a/packages/eventing-core/src/predicate.ts b/packages/eventing-core/src/predicate.ts new file mode 100644 index 000000000..d63a7b201 --- /dev/null +++ b/packages/eventing-core/src/predicate.ts @@ -0,0 +1,414 @@ +import type { + FieldRef, + NormalizedSignal, + SignalLiteral, + SignalPredicate, + SignalProjectionSpec, + SignalScalar, + SignalScalarType, +} from "./model" +import { + fieldKey, + MAX_DECIMAL_INT64_LENGTH, + MAX_IN_VALUES, + MAX_PREDICATE_DEPTH, + MAX_PREDICATE_NODES, + MAX_STRING_LITERAL_BYTES, +} from "./model" + +const INT64_MIN = -(1n << 63n) +const INT64_MAX = (1n << 63n) - 1n +const ORDERED_TYPES = new Set(["int64", "float64", "timestamp", "duration"]) + +export interface ValidationIssue { + readonly path: string + readonly message: string +} + +export class SignalPredicateValidationError extends Error { + readonly issues: readonly ValidationIssue[] + + constructor(issues: readonly ValidationIssue[]) { + super(issues.map(({ path, message }) => `${path}: ${message}`).join("; ")) + this.name = "SignalPredicateValidationError" + this.issues = issues + } +} + +const stringBytes = (value: string): number => new TextEncoder().encode(value).byteLength + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const assertScalarInputBudget = (value: unknown): void => { + if (!isRecord(value) || typeof value.type !== "string") return + if (value.type === "string" && typeof value.value === "string") { + if (stringBytes(value.value) > MAX_STRING_LITERAL_BYTES) + throw new Error(`selector string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`) + return + } + if ( + (value.type === "int64" || value.type === "duration") && + typeof value.value === "string" && + value.value.length > MAX_DECIMAL_INT64_LENGTH + ) + throw new Error(`${value.type} literal exceeds ${MAX_DECIMAL_INT64_LENGTH} characters`) +} + +/** + * Reject hostile selector topology before the recursive runtime schema sees it. + * HTTP hosts should additionally bound the serialized request body. + */ +export const assertSignalProjectionInputBudget = (candidate: unknown): void => { + if (!isRecord(candidate) || candidate.selector === undefined) return + const stack: Array<{ readonly value: unknown; readonly depth: number }> = [ + { value: candidate.selector, depth: 1 }, + ] + const seen = new Set() + let nodes = 0 + while (stack.length > 0) { + const current = stack.pop()! + if (current.depth > MAX_PREDICATE_DEPTH) + throw new Error(`predicate depth exceeds ${MAX_PREDICATE_DEPTH}`) + nodes += 1 + if (nodes > MAX_PREDICATE_NODES) throw new Error(`predicate exceeds ${MAX_PREDICATE_NODES} nodes`) + if (!isRecord(current.value)) continue + if (seen.has(current.value)) throw new Error("predicate must be acyclic JSON") + seen.add(current.value) + + switch (current.value.op) { + case "all": + case "any": { + const clauses = current.value.clauses + if (!Array.isArray(clauses)) break + if (clauses.length > MAX_PREDICATE_NODES) + throw new Error(`predicate clause list exceeds ${MAX_PREDICATE_NODES} entries`) + for (let index = clauses.length - 1; index >= 0; index--) + stack.push({ value: clauses[index], depth: current.depth + 1 }) + break + } + case "not": + stack.push({ value: current.value.clause, depth: current.depth + 1 }) + break + case "in": { + const values = current.value.values + if (!Array.isArray(values)) break + if (values.length > MAX_IN_VALUES) throw new Error(`in exceeds ${MAX_IN_VALUES} values`) + for (const value of values) assertScalarInputBudget(value) + break + } + default: + assertScalarInputBudget(current.value.value) + } + } +} + +const parseInt64 = (value: string): bigint | null => { + try { + const parsed = BigInt(value) + return parsed >= INT64_MIN && parsed <= INT64_MAX ? parsed : null + } catch { + return null + } +} + +const isLeapYear = (year: number): boolean => year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) + +const daysInMonth = (year: number, month: number): number => { + switch (month) { + case 2: + return isLeapYear(year) ? 29 : 28 + case 4: + case 6: + case 9: + case 11: + return 30 + default: + return 31 + } +} + +const TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|([+-])(\d{2}):(\d{2}))$/ + +/** Parse the v1 RFC 3339 subset into exact UTC nanoseconds. */ +export const timestampToEpochNanos = (value: string): bigint | null => { + const match = TIMESTAMP.exec(value) + if (!match) return null + const year = Number(match[1]) + const month = Number(match[2]) + const day = Number(match[3]) + const hour = Number(match[4]) + const minute = Number(match[5]) + const second = Number(match[6]) + const fraction = match[7] ?? "" + if ( + month < 1 || + month > 12 || + day < 1 || + day > daysInMonth(year, month) || + hour > 23 || + minute > 59 || + second > 59 + ) + return null + + let offsetMinutes = 0 + if (match[8] !== "Z") { + const offsetHours = Number(match[10]) + const offsetMinutePart = Number(match[11]) + if (offsetHours > 23 || offsetMinutePart > 59) return null + offsetMinutes = offsetHours * 60 + offsetMinutePart + if (match[9] === "-") offsetMinutes = -offsetMinutes + } + + const date = new Date(0) + date.setUTCFullYear(year, month - 1, day) + date.setUTCHours(hour, minute, second, 0) + const milliseconds = date.getTime() - offsetMinutes * 60_000 + if (!Number.isFinite(milliseconds)) return null + const nanos = BigInt(fraction.padEnd(9, "0")) + return BigInt(milliseconds) * 1_000_000n + nanos +} + +export const validateSignalScalar = (scalar: SignalScalar, path = "value"): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + switch (scalar.type) { + case "string": + case "boolean": + break + case "int64": + case "duration": + if (parseInt64(scalar.value) === null) + issues.push({ path, message: `${scalar.type} must be a signed 64-bit decimal integer` }) + break + case "float64": + if (!Number.isFinite(scalar.value)) issues.push({ path, message: "float64 must be finite" }) + break + case "timestamp": + if (timestampToEpochNanos(scalar.value) === null) + issues.push({ + path, + message: "timestamp must be a valid RFC 3339 instant with an explicit offset", + }) + break + } + return issues +} + +export const validateSignalLiteral = (literal: SignalLiteral, path = "value"): readonly ValidationIssue[] => [ + ...validateSignalScalar(literal, path), + ...(literal.type === "string" && stringBytes(literal.value) > MAX_STRING_LITERAL_BYTES + ? [{ path, message: `string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes` }] + : []), +] + +export const validateSignalPredicate = (predicate: SignalPredicate): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + let nodes = 0 + + const visit = (node: SignalPredicate, path: string, depth: number): void => { + nodes += 1 + if (nodes > MAX_PREDICATE_NODES) return + if (depth > MAX_PREDICATE_DEPTH) { + issues.push({ path, message: `predicate depth exceeds ${MAX_PREDICATE_DEPTH}` }) + return + } + + switch (node.op) { + case "all": + case "any": + if (node.clauses.length === 0) + issues.push({ + path: `${path}.clauses`, + message: `${node.op} requires at least one clause`, + }) + for (let i = 0; i < node.clauses.length; i++) + visit(node.clauses[i]!, `${path}.clauses[${i}]`, depth + 1) + break + case "not": + visit(node.clause, `${path}.clause`, depth + 1) + break + case "exists": + break + case "contains": + if (node.field.type !== "string" || node.value.type !== "string") + issues.push({ path, message: "contains requires a string field and string literal" }) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) + break + case "gt": + case "gte": + case "lt": + case "lte": + if (!ORDERED_TYPES.has(node.field.type)) + issues.push({ path, message: `${node.op} is not supported for ${node.field.type}` }) + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) + break + case "eq": + case "neq": + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) + break + case "in": + if (node.values.length === 0) + issues.push({ path: `${path}.values`, message: "in requires at least one value" }) + if (node.values.length > MAX_IN_VALUES) + issues.push({ path: `${path}.values`, message: `in exceeds ${MAX_IN_VALUES} values` }) + for (let i = 0; i < node.values.length; i++) { + const value = node.values[i]! + if (value.type !== node.field.type) + issues.push({ + path: `${path}.values[${i}]`, + message: "field and literal types must match", + }) + issues.push(...validateSignalLiteral(value, `${path}.values[${i}]`)) + } + break + } + } + + visit(predicate, "selector", 1) + if (nodes > MAX_PREDICATE_NODES) + issues.push({ path: "selector", message: `predicate exceeds ${MAX_PREDICATE_NODES} nodes` }) + return issues +} + +export const assertValidSignalPredicate = (predicate: SignalPredicate): void => { + const issues = validateSignalPredicate(predicate) + if (issues.length > 0) throw new SignalPredicateValidationError(issues) +} + +export const validateSignalProjectionSpec = ( + projection: SignalProjectionSpec, +): readonly ValidationIssue[] => [ + ...(timestampToEpochNanos(projection.activeFrom) === null + ? [{ path: "activeFrom", message: "must be a valid RFC 3339 instant with an explicit offset" }] + : []), + ...validateSignalPredicate(projection.selector), +] + +export interface PredicateEvaluation { + readonly matches: boolean + readonly typeMismatches: readonly FieldRef[] +} + +const scalarEquals = (left: SignalScalar, right: SignalScalar): boolean => { + if (left.type !== right.type) return false + switch (left.type) { + case "string": + return right.type === "string" && left.value === right.value + case "boolean": + return right.type === "boolean" && left.value === right.value + case "float64": + return right.type === "float64" && left.value === right.value + case "int64": + return right.type === "int64" && BigInt(left.value) === BigInt(right.value) + case "duration": + return right.type === "duration" && BigInt(left.value) === BigInt(right.value) + case "timestamp": + return ( + right.type === "timestamp" && + timestampToEpochNanos(left.value) === timestampToEpochNanos(right.value) + ) + } +} + +const scalarOrder = (left: SignalScalar, right: SignalScalar): number | null => { + if (left.type !== right.type || !ORDERED_TYPES.has(left.type)) return null + switch (left.type) { + case "int64": { + if (right.type !== "int64") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "duration": { + if (right.type !== "duration") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "float64": + return right.type !== "float64" + ? null + : left.value < right.value + ? -1 + : left.value > right.value + ? 1 + : 0 + case "timestamp": { + if (right.type !== "timestamp") return null + const a = timestampToEpochNanos(left.value)! + const b = timestampToEpochNanos(right.value)! + return a < b ? -1 : a > b ? 1 : 0 + } + default: + return null + } +} + +export type CompiledSignalPredicate = (signal: NormalizedSignal) => PredicateEvaluation + +export const compileSignalPredicate = (predicate: SignalPredicate): CompiledSignalPredicate => { + assertValidSignalPredicate(predicate) + + return (signal) => { + const typeMismatches: FieldRef[] = [] + const readField = (field: FieldRef): SignalScalar | undefined => { + const value = signal.fields.get(fieldKey(field)) + if (value === undefined) return undefined + if (value.type !== field.type || validateSignalScalar(value).length > 0) { + typeMismatches.push(field) + return undefined + } + return value + } + + const evaluate = (node: SignalPredicate): boolean => { + switch (node.op) { + case "all": + return node.clauses.every(evaluate) + case "any": + return node.clauses.some(evaluate) + case "not": + return !evaluate(node.clause) + case "exists": { + return readField(node.field) !== undefined + } + case "eq": + case "neq": + case "gt": + case "gte": + case "lt": + case "lte": + case "contains": { + const value = readField(node.field) + if (value === undefined) return false + if (node.op === "eq") return scalarEquals(value, node.value) + if (node.op === "neq") return !scalarEquals(value, node.value) + if (node.op === "contains") + return ( + value.type === "string" && + node.value.type === "string" && + value.value.includes(node.value.value) + ) + const order = scalarOrder(value, node.value) + if (order === null) return false + if (node.op === "gt") return order > 0 + if (node.op === "gte") return order >= 0 + if (node.op === "lt") return order < 0 + return order <= 0 + } + case "in": { + const value = readField(node.field) + if (value === undefined) return false + return node.values.some((candidate) => scalarEquals(value, candidate)) + } + } + } + + return { matches: evaluate(predicate), typeMismatches } + } +} diff --git a/packages/eventing-core/src/registry.test.ts b/packages/eventing-core/src/registry.test.ts new file mode 100644 index 000000000..e8fe96d69 --- /dev/null +++ b/packages/eventing-core/src/registry.test.ts @@ -0,0 +1,325 @@ +import { describe, expect, it } from "vitest" +import { + CompiledProjectionRegistry, + canonicalJson, + defineSignalFields, + makeEventId, + MAX_CLOUD_EVENT_BYTES, + ProjectorRegistry, + SignalSourceRegistry, + validateMapleCloudEvent, + type NormalizedSignal, + type SignalProjectionSpec, +} from "./index" + +const signal = (overrides: Partial = {}): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "event-123", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00.123456789Z", + observedAt: "2026-08-07T19:42:01Z", + subject: "project/example/issues/42", + fields: defineSignalFields([ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + ]), + data: { issue: { iid: 42, title: "Example" } }, + ...overrides, +}) + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 3, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + projector: { id: "gitlab.issue", version: 1, config: { includeTitle: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const projectors = (): ProjectorRegistry => + new ProjectorRegistry().register({ + id: "gitlab.issue", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.created.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue:v1", + decodeConfig: (value) => { + if (typeof value !== "object" || value === null) throw new Error("invalid projector config") + return value + }, + project: (input) => ({ data: input.data as { issue: { iid: number; title: string } } }), + }) + +const sources = (): SignalSourceRegistry => + new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + openFields: [ + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64", "timestamp", "duration"], + operators: ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + +describe("CompiledProjectionRegistry", () => { + it("canonicalizes JSON independently of object insertion order", () => { + expect(canonicalJson({ z: 1, nested: { b: true, a: [2, 1] }, a: "first" })).toBe( + '{"a":"first","nested":{"a":[2,1],"b":true},"z":1}', + ) + const shared = { value: 1 } + expect(canonicalJson({ left: shared, right: shared })).toBe( + '{"left":{"value":1},"right":{"value":1}}', + ) + const cyclic: { self?: unknown } = {} + cyclic.self = cyclic + expect(() => canonicalJson(cyclic as never)).toThrow("finite acyclic JSON") + expect(() => canonicalJson({ invalid: Number.NaN })).toThrow("finite acyclic JSON") + }) + + it("projects every match into a deterministic CloudEvent", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const first = registry.evaluate(signal()) + const second = registry.evaluate(signal()) + expect(first.failures).toEqual([]) + expect(first.events).toEqual(second.events) + expect(first.events).toHaveLength(1) + expect(first.events[0]).toMatchObject({ + specversion: "1.0", + id: makeEventId({ + tenantId: "tenant-a", + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + occurrenceId: "event-123", + projectionId: "gitlab-issue-created", + projectionRevision: 3, + }), + type: "dev.maple.gitlab.issue.created.v1", + subject: "project/example/issues/42", + projectionrevision: 3, + sourceoccurrenceid: "event-123", + sourceidentityquality: "source", + data: signal().data, + }) + }) + + it("validates historical CloudEvents that predate source identity extensions", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const event = registry.evaluate(signal()).events[0]! + const { sourceoccurrenceid: _occurrence, sourceidentityquality: _quality, ...historical } = event + const validated = validateMapleCloudEvent(historical).event + expect(validated.id).toBe(event.id) + expect(validated.sourceoccurrenceid).toBeUndefined() + expect(validated.sourceidentityquality).toBeUndefined() + }) + + it("runs every matching projection from one immutable registry snapshot", () => { + const registry = CompiledProjectionRegistry.compile( + [projection(), projection({ id: "gitlab-issue-created-audit" })], + sources(), + projectors(), + ) + const result = registry.evaluate(signal()) + expect(result.failures).toEqual([]) + expect(result.events.map(({ projectionid }) => projectionid)).toEqual([ + "gitlab-issue-created", + "gitlab-issue-created-audit", + ]) + }) + + it("runs all matching projections and isolates projector failures", () => { + const registryDefinitions = projectors().register({ + id: "broken", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.broken.v1", + dataSchema: "urn:maple:event-schema:broken:v1", + decodeConfig: () => ({}), + project: () => { + throw new Error("projector invariant failed") + }, + }) + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "broken-projection", projector: { id: "broken", version: 1, config: {} } }), + ], + sources(), + registryDefinitions, + ) + const result = registry.evaluate(signal()) + expect(result.events).toHaveLength(1) + expect(result.failures).toEqual([ + expect.objectContaining({ + projectionId: "broken-projection", + message: "projector invariant failed", + }), + ]) + }) + + it("isolates complete-envelope schema and size failures from successful siblings", () => { + const registryDefinitions = projectors() + .register({ + id: "oversized", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.oversized.v1", + dataSchema: "urn:maple:event-schema:oversized:v1", + decodeConfig: () => ({}), + project: () => ({ data: { payload: "x".repeat(MAX_CLOUD_EVENT_BYTES) } }), + }) + .register({ + id: "invalid-envelope", + version: 1, + sourceKinds: ["otel.log"], + outputType: "x".repeat(257), + dataSchema: "urn:maple:event-schema:invalid-envelope:v1", + decodeConfig: () => ({}), + project: () => ({ data: {} }), + }) + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ + id: "oversized-projection", + projector: { id: "oversized", version: 1, config: {} }, + }), + projection({ + id: "invalid-envelope-projection", + projector: { id: "invalid-envelope", version: 1, config: {} }, + }), + ], + sources(), + registryDefinitions, + ) + const result = registry.evaluate(signal()) + expect(result.events.map(({ projectionid }) => projectionid)).toEqual(["gitlab-issue-created"]) + expect(result.failures).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + projectionId: "oversized-projection", + message: expect.stringContaining("CloudEvent exceeds"), + }), + expect.objectContaining({ + projectionId: "invalid-envelope-projection", + }), + ]), + ) + }) + + it("isolates tenants, source kinds, activation time, and disabled revisions", () => { + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "future", revision: 1, activeFrom: "2026-08-08T00:00:00Z" }), + projection({ id: "disabled", revision: 1, enabled: false }), + projection({ id: "other-tenant", revision: 1, tenantId: "tenant-b" }), + ], + sources(), + projectors(), + ) + expect(registry.evaluate(signal()).events.map(({ projectionid }) => projectionid)).toEqual([ + "gitlab-issue-created", + ]) + expect(registry.evaluate(signal({ sourceKind: "otel.span" })).events).toEqual([]) + }) + + it("requires occurrence identity for durable projection", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const result = registry.evaluate(signal({ occurrenceId: null, identityQuality: "none" })) + expect(result.events).toEqual([]) + expect(result.failures[0]?.message).toBe( + "durable event projection requires stable or derived occurrence identity", + ) + }) + + it("rejects duplicate registrations, projection revisions, and invalid projector bindings", () => { + const definitions = projectors() + expect(() => + definitions.register({ + id: "gitlab.issue", + version: 1, + sourceKinds: ["otel.log"], + outputType: "duplicate", + dataSchema: "duplicate", + decodeConfig: (value) => value, + project: () => ({ data: {} }), + }), + ).toThrow("duplicate projector registration") + expect(() => + CompiledProjectionRegistry.compile([projection(), projection()], sources(), projectors()), + ).toThrow("duplicate projection revision") + expect(() => + CompiledProjectionRegistry.compile( + [projection({ projector: { id: "missing", version: 1, config: {} } })], + sources(), + projectors(), + ), + ).toThrow("unregistered projector") + }) + + it("validates selector fields and operators against the source catalog", () => { + const closed = new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["eq"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "contains", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("contains is not allowed for catalog field") + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "eq", + field: { namespace: "attribute", key: "unknown", type: "string" }, + value: { type: "string", value: "x" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("unknown field attribute:unknown") + }) +}) diff --git a/packages/eventing-core/src/registry.ts b/packages/eventing-core/src/registry.ts new file mode 100644 index 000000000..92c5bb7f0 --- /dev/null +++ b/packages/eventing-core/src/registry.ts @@ -0,0 +1,186 @@ +import { makeCloudEvent } from "./event" +import { Schema } from "effect" +import type { + JsonValue, + MapleCloudEvent, + NormalizedSignal, + ProjectedEventData, + SignalProjectionSpec, +} from "./model" +import { SignalProjectionSpecSchema } from "./model" +import { timestampToEpochNanos, compileSignalPredicate, validateSignalProjectionSpec } from "./predicate" +import { SignalSourceRegistry, validatePredicateAgainstSource } from "./source" + +export interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly decodeConfig: (value: unknown) => TConfig + readonly project: (signal: NormalizedSignal, config: TConfig) => ProjectedEventData +} + +type ErasedSignalProjector = SignalProjector + +export class ProjectorRegistry { + readonly #projectors = new Map() + + register(projector: SignalProjector): this { + if (projector.id.trim().length === 0) throw new Error("projector ID must not be empty") + if (!Number.isSafeInteger(projector.version) || projector.version < 1) + throw new Error("projector version must be a positive safe integer") + if (projector.sourceKinds.length === 0) throw new Error("projector must accept a source kind") + if (projector.outputType.trim().length === 0) + throw new Error("projector output type must not be empty") + if (projector.dataSchema.trim().length === 0) + throw new Error("projector data schema must not be empty") + const key = ProjectorRegistry.key(projector.id, projector.version) + if (this.#projectors.has(key)) throw new Error(`duplicate projector registration: ${key}`) + this.#projectors.set(key, projector as ErasedSignalProjector) + return this + } + + get(id: string, version: number): ErasedSignalProjector | undefined { + return this.#projectors.get(ProjectorRegistry.key(id, version)) + } + + static key(id: string, version: number): string { + return `${id}@${version}` + } +} + +interface CompiledProjection { + readonly spec: SignalProjectionSpec + readonly evaluate: ReturnType + readonly projector: ErasedSignalProjector + readonly config: unknown + readonly activeFromNanos: bigint +} + +export interface ProjectionFailure { + readonly projectionId: string + readonly projectionRevision: number + readonly occurrenceId: string | null + readonly message: string +} + +export interface ProjectionBatchResult { + readonly events: readonly MapleCloudEvent[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +const validateProjectedData = (value: ProjectedEventData): ProjectedEventData => { + if (value.time !== undefined && timestampToEpochNanos(value.time) === null) + throw new Error("projector returned an invalid event timestamp") + return value +} + +/** Immutable compiled snapshot. Hosts atomically replace the whole instance. */ +export class CompiledProjectionRegistry { + readonly #bySourceKind: ReadonlyMap + + private constructor(bySourceKind: ReadonlyMap) { + this.#bySourceKind = bySourceKind + } + + static compile( + specs: readonly SignalProjectionSpec[], + sources: SignalSourceRegistry, + projectors: ProjectorRegistry, + ): CompiledProjectionRegistry { + const bySourceKind = new Map() + const revisions = new Set() + + for (const candidate of specs) { + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + const source = sources.get(spec.sourceKind) + if (!source) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered source ${spec.sourceKind}`, + ) + const issues = [ + ...validateSignalProjectionSpec(spec), + ...validatePredicateAgainstSource(spec.selector, source), + ] + if (issues.length > 0) + throw new Error( + `invalid projection ${spec.id}@${spec.revision}: ${issues + .map(({ path, message }) => `${path}: ${message}`) + .join("; ")}`, + ) + const revisionKey = `${spec.tenantId}:${spec.id}@${spec.revision}` + if (revisions.has(revisionKey)) throw new Error(`duplicate projection revision: ${revisionKey}`) + revisions.add(revisionKey) + if (!spec.enabled) continue + + const projector = projectors.get(spec.projector.id, spec.projector.version) + if (!projector) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered projector ${spec.projector.id}@${spec.projector.version}`, + ) + if (!projector.sourceKinds.includes(spec.sourceKind)) + throw new Error( + `projector ${projector.id}@${projector.version} does not accept ${spec.sourceKind}`, + ) + + const compiled: CompiledProjection = { + spec, + evaluate: compileSignalPredicate(spec.selector), + projector, + config: projector.decodeConfig(spec.projector.config), + activeFromNanos: timestampToEpochNanos(spec.activeFrom)!, + } + const bucket = bySourceKind.get(spec.sourceKind) + if (bucket) bucket.push(compiled) + else bySourceKind.set(spec.sourceKind, [compiled]) + } + + return new CompiledProjectionRegistry(bySourceKind) + } + + evaluate(signal: NormalizedSignal): ProjectionBatchResult { + const events: MapleCloudEvent[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + const observedAtNanos = timestampToEpochNanos(signal.observedAt) + + for (const projection of this.#bySourceKind.get(signal.sourceKind) ?? []) { + if (projection.spec.tenantId !== signal.tenantId) continue + if (observedAtNanos === null || observedAtNanos < projection.activeFromNanos) continue + const evaluation = projection.evaluate(signal) + for (const field of evaluation.typeMismatches) + typeMismatchFields.add(`${field.namespace}:${field.key}`) + if (!evaluation.matches) continue + + try { + const projected = validateProjectedData( + projection.projector.project(signal, projection.config), + ) + events.push( + makeCloudEvent({ + signal, + projection: projection.spec, + projectorId: projection.projector.id, + projectorVersion: projection.projector.version, + outputType: projection.projector.outputType, + dataSchema: projection.projector.dataSchema, + subject: projected.subject, + time: projected.time, + data: projected.data as JsonValue, + }), + ) + } catch (error) { + failures.push({ + projectionId: projection.spec.id, + projectionRevision: projection.spec.revision, + occurrenceId: signal.occurrenceId, + message: error instanceof Error ? error.message : String(error), + }) + } + } + + return { events, failures, typeMismatchFields: [...typeMismatchFields] } + } +} diff --git a/packages/eventing-core/src/source.ts b/packages/eventing-core/src/source.ts new file mode 100644 index 000000000..cc1a722f7 --- /dev/null +++ b/packages/eventing-core/src/source.ts @@ -0,0 +1,157 @@ +import type { FieldNamespace, FieldRef, NormalizedSignal, SignalPredicate, SignalScalarType } from "./model" +import { fieldKey } from "./model" +import type { ValidationIssue } from "./predicate" + +export type SignalLeafOperator = "exists" | "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" | "in" +export type ReplayCapability = "exact" | "coerced" | "unavailable" + +interface SignalFieldCatalogEntryBase { + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export type SignalFieldCatalogEntry = SignalFieldCatalogEntryBase & + ( + | { readonly field: FieldRef; readonly types?: never } + | { + readonly field: Pick + readonly types: readonly SignalScalarType[] + } + ) + +export interface OpenFieldNamespacePolicy { + readonly namespace: FieldNamespace + readonly types: readonly SignalScalarType[] + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export interface SignalSourceDefinition { + readonly sourceKind: string + readonly fields: readonly SignalFieldCatalogEntry[] + readonly openFields?: readonly OpenFieldNamespacePolicy[] +} + +export interface SignalSourceAdapter { + readonly definition: SignalSourceDefinition + readonly normalize: (raw: TRaw, context: TContext) => readonly NormalizedSignal[] +} + +interface RegisteredSignalSource { + readonly definition: SignalSourceDefinition + readonly fields: ReadonlyMap + readonly openFields: ReadonlyMap +} + +const catalogEntryTypes = (entry: SignalFieldCatalogEntry): readonly SignalScalarType[] => { + if (entry.types !== undefined) return entry.types + return [entry.field.type] +} + +export class SignalSourceRegistry { + readonly #sources = new Map() + + register(definition: SignalSourceDefinition): this { + if (definition.sourceKind.trim().length === 0) throw new Error("source kind must not be empty") + if (this.#sources.has(definition.sourceKind)) + throw new Error(`duplicate source registration: ${definition.sourceKind}`) + + const fields = new Map() + for (const entry of definition.fields) { + const key = fieldKey(entry.field) + if (fields.has(key)) + throw new Error(`duplicate field catalog entry: ${definition.sourceKind}:${key}`) + if (entry.operators.length === 0) throw new Error(`field catalog entry has no operators: ${key}`) + if (entry.types !== undefined && entry.types.length === 0) + throw new Error(`field catalog entry has no types: ${key}`) + fields.set(key, entry) + } + + const openFields = new Map() + for (const policy of definition.openFields ?? []) { + if (openFields.has(policy.namespace)) + throw new Error(`duplicate open field policy: ${definition.sourceKind}:${policy.namespace}`) + if (policy.types.length === 0 || policy.operators.length === 0) + throw new Error(`open field policy must declare types and operators: ${policy.namespace}`) + openFields.set(policy.namespace, policy) + } + + this.#sources.set(definition.sourceKind, { definition, fields, openFields }) + return this + } + + get(sourceKind: string): RegisteredSignalSource | undefined { + return this.#sources.get(sourceKind) + } +} + +const leafFields = ( + predicate: SignalPredicate, +): ReadonlyArray<{ + readonly field: FieldRef + readonly operator: SignalLeafOperator + readonly path: string +}> => { + const fields: Array<{ field: FieldRef; operator: SignalLeafOperator; path: string }> = [] + const visit = (node: SignalPredicate, path: string): void => { + switch (node.op) { + case "all": + case "any": + for (let i = 0; i < node.clauses.length; i++) visit(node.clauses[i]!, `${path}.clauses[${i}]`) + break + case "not": + visit(node.clause, `${path}.clause`) + break + default: + fields.push({ field: node.field, operator: node.op, path }) + } + } + visit(predicate, "selector") + return fields +} + +export const validatePredicateAgainstSource = ( + predicate: SignalPredicate, + source: RegisteredSignalSource, +): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + for (const leaf of leafFields(predicate)) { + const catalogEntry = source.fields.get(fieldKey(leaf.field)) + if (catalogEntry) { + const catalogTypes = catalogEntryTypes(catalogEntry) + if (!catalogTypes.includes(leaf.field.type)) + issues.push({ + path: `${leaf.path}.field.type`, + message: `catalog field ${fieldKey(leaf.field)} allows ${catalogTypes.join(", ")}`, + }) + if (!catalogEntry.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for catalog field ${fieldKey(leaf.field)}`, + }) + continue + } + + const open = source.openFields.get(leaf.field.namespace) + if (!open) { + issues.push({ + path: `${leaf.path}.field`, + message: `unknown field ${fieldKey(leaf.field)} for source ${source.definition.sourceKind}`, + }) + continue + } + if (!open.types.includes(leaf.field.type)) + issues.push({ + path: `${leaf.path}.field.type`, + message: `${leaf.field.type} is not allowed for open ${leaf.field.namespace} fields`, + }) + if (!open.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for open ${leaf.field.namespace} fields`, + }) + } + return issues +} diff --git a/packages/eventing-core/tsconfig.json b/packages/eventing-core/tsconfig.json new file mode 100644 index 000000000..12d9920b4 --- /dev/null +++ b/packages/eventing-core/tsconfig.json @@ -0,0 +1,23 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022", "DOM"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true, + "plugins": [ + { + "name": "@effect/language-service", + "reportSuggestionsAsWarningsInTsc": true + } + ] + } +}