From 560a46af24bc18ab8ff1d795730c2d8130d68277 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Fri, 4 Sep 2026 21:08:32 -0700 Subject: [PATCH 1/2] feat(growth): link install and development runtime evidence to founder outreach --- apps/lifecycle/README.md | 8 + apps/lifecycle/src/app/dispatch/index.ts | 1 + apps/lifecycle/src/campaign/send.spec.ts | 160 +++++- apps/lifecycle/src/campaign/send.ts | 18 +- apps/lifecycle/src/dispatcher.spec.ts | 79 +++ apps/lifecycle/src/dispatcher.ts | 16 + apps/lifecycle/src/email-keyring.spec.ts | 68 +++ apps/lifecycle/src/email-keyring.ts | 67 +++ apps/website/next.config.ts | 2 + .../growth/collect/v1/[source]/route.spec.ts | 29 ++ .../api/growth/collect/v1/[source]/route.ts | 14 + .../src/lib/growth/collection-route.spec.ts | 401 +++++++++++++++ .../src/lib/growth/collection-route.ts | 224 +++++++++ .../lib/growth/runtime-announcements.spec.ts | 132 +++++ .../src/lib/growth/runtime-announcements.ts | 116 +++++ .../website/src/lib/growth/website-content.ts | 53 ++ .../src/lib/growth/website-metadata.spec.ts | 58 +++ .../src/lib/growth/website-metadata.ts | 61 +++ libs/growth/project.json | 1 + libs/growth/src/index.ts | 32 ++ libs/growth/src/lib/contacts.spec.ts | 223 +++++++++ libs/growth/src/lib/contacts.ts | 186 +++++++ libs/growth/src/lib/jobs.spec.ts | 6 + libs/growth/src/lib/jobs.ts | 53 +- .../src/lib/observability/admission.spec.ts | 27 ++ .../growth/src/lib/observability/admission.ts | 80 +++ .../src/lib/observability/canonical.spec.ts | 24 + .../growth/src/lib/observability/canonical.ts | 42 ++ .../src/lib/observability/contracts.spec.ts | 235 +++++++++ .../growth/src/lib/observability/contracts.ts | 357 ++++++++++++++ .../lib/observability/enrichment-contract.ts | 16 + .../src/lib/observability/form-projection.ts | 130 +++++ libs/growth/src/lib/observability/ingest.ts | 176 +++++++ .../install-runtime-contract.spec.ts | 60 +++ .../src/lib/observability/install-runtime.ts | 110 +++++ .../src/lib/observability/projection.ts | 203 ++++++++ libs/growth/src/lib/observability/queries.ts | 136 ++++++ .../growth/src/lib/observability/redaction.ts | 214 ++++++++ libs/growth/src/lib/observability/replay.ts | 96 ++++ libs/growth/src/lib/observability/store.ts | 40 ++ libs/growth/src/lib/stops.spec.ts | 6 + libs/growth/test/contacts.integration.spec.ts | 3 + .../form-observations.integration.spec.ts | 227 +++++++++ .../test/install-runtime.integration.spec.ts | 459 ++++++++++++++++++ libs/growth/test/jobs.integration.spec.ts | 2 + .../test/migrations.integration.spec.ts | 19 +- ...bservability-admission.integration.spec.ts | 80 +++ libs/growth/test/observability-fixtures.ts | 85 ++++ .../observability-ingest.integration.spec.ts | 113 +++++ .../observability-journey.integration.spec.ts | 245 ++++++++++ ...servability-projection.integration.spec.ts | 242 +++++++++ ...bservability-redaction.integration.spec.ts | 356 ++++++++++++++ .../observability-schema.integration.spec.ts | 94 ++++ libs/growth/vite.integration.config.mts | 3 + libs/growth/vite.operator-cli.config.mts | 1 + migrations/0004_growth_observability.sql | 93 ++++ .../0005_growth_observability_views.sql | 30 ++ migrations/0006_growth_form_observations.sql | 21 + migrations/0007_growth_install_runtime.sql | 15 + package.json | 1 + scripts/growth-observability.mts | 257 ++++++++++ scripts/growth-observability.spec.ts | 109 +++++ 62 files changed, 6407 insertions(+), 8 deletions(-) create mode 100644 apps/lifecycle/src/email-keyring.spec.ts create mode 100644 apps/lifecycle/src/email-keyring.ts create mode 100644 apps/website/src/app/api/growth/collect/v1/[source]/route.spec.ts create mode 100644 apps/website/src/app/api/growth/collect/v1/[source]/route.ts create mode 100644 apps/website/src/lib/growth/collection-route.spec.ts create mode 100644 apps/website/src/lib/growth/collection-route.ts create mode 100644 apps/website/src/lib/growth/runtime-announcements.spec.ts create mode 100644 apps/website/src/lib/growth/runtime-announcements.ts create mode 100644 apps/website/src/lib/growth/website-content.ts create mode 100644 apps/website/src/lib/growth/website-metadata.spec.ts create mode 100644 apps/website/src/lib/growth/website-metadata.ts create mode 100644 libs/growth/src/lib/observability/admission.spec.ts create mode 100644 libs/growth/src/lib/observability/admission.ts create mode 100644 libs/growth/src/lib/observability/canonical.spec.ts create mode 100644 libs/growth/src/lib/observability/canonical.ts create mode 100644 libs/growth/src/lib/observability/contracts.spec.ts create mode 100644 libs/growth/src/lib/observability/contracts.ts create mode 100644 libs/growth/src/lib/observability/enrichment-contract.ts create mode 100644 libs/growth/src/lib/observability/form-projection.ts create mode 100644 libs/growth/src/lib/observability/ingest.ts create mode 100644 libs/growth/src/lib/observability/install-runtime-contract.spec.ts create mode 100644 libs/growth/src/lib/observability/install-runtime.ts create mode 100644 libs/growth/src/lib/observability/projection.ts create mode 100644 libs/growth/src/lib/observability/queries.ts create mode 100644 libs/growth/src/lib/observability/redaction.ts create mode 100644 libs/growth/src/lib/observability/replay.ts create mode 100644 libs/growth/src/lib/observability/store.ts create mode 100644 libs/growth/test/form-observations.integration.spec.ts create mode 100644 libs/growth/test/install-runtime.integration.spec.ts create mode 100644 libs/growth/test/observability-admission.integration.spec.ts create mode 100644 libs/growth/test/observability-fixtures.ts create mode 100644 libs/growth/test/observability-ingest.integration.spec.ts create mode 100644 libs/growth/test/observability-journey.integration.spec.ts create mode 100644 libs/growth/test/observability-projection.integration.spec.ts create mode 100644 libs/growth/test/observability-redaction.integration.spec.ts create mode 100644 libs/growth/test/observability-schema.integration.spec.ts create mode 100644 migrations/0004_growth_observability.sql create mode 100644 migrations/0005_growth_observability_views.sql create mode 100644 migrations/0006_growth_form_observations.sql create mode 100644 migrations/0007_growth_install_runtime.sql create mode 100644 scripts/growth-observability.mts create mode 100644 scripts/growth-observability.spec.ts diff --git a/apps/lifecycle/README.md b/apps/lifecycle/README.md index 926df216c..0a830e3cf 100644 --- a/apps/lifecycle/README.md +++ b/apps/lifecycle/README.md @@ -9,6 +9,14 @@ The service has two database boundaries: Neither variable falls back to the other. Preview and production must use different Neon resources for both boundaries. Configure no lifecycle secret with a `NEXT_PUBLIC_` prefix. +Install/runtime activation has a separate rollout switch: `GROWTH_INSTALL_RUNTIME_HELLO_ENABLED` defaults to `false` and accepts only exact `true` or `false`. Only when it and campaign enrollment are enabled does the existing lifecycle tick resolve linked activations before materializing the campaign cohort. Configure the same server-only `GROWTH_EMAIL_HMAC_ACTIVE_VERSION`, `GROWTH_EMAIL_HMAC_ACTIVE_SECRET`, and optional `GROWTH_EMAIL_HMAC_PREVIOUS_KEYS` used by collection; these keys are read lazily only for enabled activation processing. Existing form and claim enrollment needs no new HMAC configuration while the rollout switch is off. Announcement requests do not run this work or submit email. + +Apply migrations 0004–0007 before deploying the backend: contact deletion and campaign authorization use the observation tables even while the activation switch is off. Verify a second migration run applies nothing. For databases with historical deletions, run `npm run growth:observability -- initialize-redactions --limit 100` with the matching collection HMAC keys, passing each returned `nextCursor` as `--cursor` until exhausted, before enabling identity collection or activation. + +Deploy backend observation acceptance and bridge resolution with the rollout switch off. Verify the synthetic journey in preview with a controlled recipient and lifecycle's matching HMAC keys, then publish the matching collectors and enable production collection and activation gradually. Preserve the existing enrollment start timestamp, campaign, delivery, and cron controls. + +A persisted `install_runtime` enrollment reason selects the existing generic founder sequence immediately, without waiting for an enrichment artifact. All three steps stay generic even if optional research later becomes available. Form and project-claim enrollments retain their existing behavior. The shared delivery authorization, reply/suppression stops, mailbox recovery guard, unsubscribe links, and once-per-contact three-step enrollment remain in force; install-derived eligibility does not verify identity or employment. + Recipient delivery also requires `GROWTH_PUBLIC_ACTION_ORIGIN`, a server-only bare HTTPS origin for the Website deployment that owns `/api/unsubscribe`. In preview, use a dedicated public custom-domain alias for the exact Website preview deployment while keeping generated preview URLs protected; the signed action token is the application-layer authorization. In production, use the canonical Website origin. Paths, query strings, fragments, credentials, and HTTP origins are rejected. The lifecycle service uses this value only to construct opaque, contact-bound unsubscribe action URLs; it never derives the origin from a request or hardcodes the production site. Set `GROWTH_DATABASE_ENVIRONMENT` to exactly `preview`, `production`, or `test` in every process that handles verified Resend events. A verified webhook whose `environment` provider tag is missing or differs from that value is acknowledged without opening a growth transaction or changing delivery/suppression state. diff --git a/apps/lifecycle/src/app/dispatch/index.ts b/apps/lifecycle/src/app/dispatch/index.ts index 05e119055..a7555a78c 100644 --- a/apps/lifecycle/src/app/dispatch/index.ts +++ b/apps/lifecycle/src/app/dispatch/index.ts @@ -25,6 +25,7 @@ export async function workflow( batchSize: configuredBatchSize(), campaignEnabled: configuration.campaignEnabled, campaignEnrollmentEnabled: configuration.campaignEnrollmentEnabled, + installRuntimeHelloEnabled: configuration.installRuntimeHelloEnabled, campaignEnrollmentStartAt: configuration.campaignEnrollmentStartAt, signal: context.signal, }); diff --git a/apps/lifecycle/src/campaign/send.spec.ts b/apps/lifecycle/src/campaign/send.spec.ts index d3b9edf3d..0de20f422 100644 --- a/apps/lifecycle/src/campaign/send.spec.ts +++ b/apps/lifecycle/src/campaign/send.spec.ts @@ -127,6 +127,56 @@ function context( } describe('prepareCampaignMessage', () => { + it('prepares the install-runtime hello immediately without research', () => { + expect( + prepareCampaignMessage({ + context: { + ...context({ enrichmentArtifact: null }), + campaignEnrollmentReason: 'install_runtime', + }, + job: job('send_step', { campaign_version: 'v1', step: 1 }), + now: new Date('2026-09-01T12:00:00.000Z'), + unsubscribeUrl: UNSUBSCRIBE, + }) + ).toMatchObject({ status: 'ready', subject: 'A practical place to start' }); + }); + + it.each([1, 2, 3] as const)( + 'keeps install-runtime step %i generic even when research is available', + (step) => { + const prepared = prepareCampaignMessage({ + context: { ...context(), campaignEnrollmentReason: 'install_runtime' }, + job: job('send_step', { campaign_version: 'v1', step }), + now: NOW, + unsubscribeUrl: UNSUBSCRIBE, + }); + expect(prepared).toMatchObject({ + status: 'ready', + subject: [ + 'A practical place to start', + 'One debugging shortcut', + 'One last architecture note', + ][step - 1], + }); + if (prepared.status !== 'ready') throw new Error('expected ready'); + expect(prepared.text).toContain(unsubscribeActionUrlValue(UNSUBSCRIBE)); + expect(prepared.text).toContain('\n\n—\nBrian\n'); + if (step === 3) + expect(prepared.text).toContain('This is my last automated follow-up.'); + } + ); + + it('rejects a fourth install-runtime sequence step', () => { + expect(() => + prepareCampaignMessage({ + context: { ...context(), campaignEnrollmentReason: 'install_runtime' }, + job: job('send_step', { campaign_version: 'v1', step: 4 }), + now: NOW, + unsubscribeUrl: UNSUBSCRIBE, + }) + ).toThrow(DeterministicLifecycleJobError); + }); + it('renders only a closed evidence-linked angle selection deterministically', () => { const cited = artifact({ cited_signals: [ @@ -358,6 +408,93 @@ function dependencies( } describe('dispatchLifecycleAppOwnedJob', () => { + it('sends the install-runtime hello through the shared recipient boundary without research', async () => { + const deps = dependencies({ + readJobContext: vi.fn().mockResolvedValue( + context({ + campaignEnrollmentReason: 'install_runtime', + enrichmentArtifact: null, + }) + ), + }); + const send = job('send_step', { campaign_version: 'v1', step: 1 }); + await expect( + dispatchLifecycleAppOwnedJob({} as SqlExecutor, send, {}, deps) + ).resolves.toBe('completed'); + expect(deps.sendRecipient).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + jobId: send.id, + leaseToken: LEASE_TOKEN, + subject: 'A practical place to start', + unsubscribeUrl: UNSUBSCRIBE, + }), + deps.recipientPolicy + ); + expect(deps.deferJob).not.toHaveBeenCalled(); + expect(deps.fetchCompanyEvidence).not.toHaveBeenCalled(); + expect(deps.generateArtifact).not.toHaveBeenCalled(); + }); + + it.each([ + 'contact_stopped', + 'contact_unapproved', + 'contact_deleted', + ] as const)( + 'preserves the shared %s delivery stop for an install-runtime hello', + async (reason) => { + const deps = dependencies({ + readJobContext: vi.fn().mockResolvedValue( + context({ + campaignEnrollmentReason: 'install_runtime', + enrichmentArtifact: null, + }) + ), + sendRecipient: vi.fn().mockResolvedValue({ accepted: false, reason }), + }); + await expect( + dispatchLifecycleAppOwnedJob( + {} as SqlExecutor, + job('send_step', { campaign_version: 'v1', step: 1 }), + {}, + deps + ) + ).resolves.toBe('cancelled'); + expect(deps.cancelJob).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ errorCode: reason }) + ); + expect(deps.generateArtifact).not.toHaveBeenCalled(); + } + ); + + it.each(['campaign_disabled', 'delivery_disabled'] as const)( + 'keeps an install-runtime hello deferred while %s', + async (reason) => { + const deps = dependencies({ + readJobContext: vi.fn().mockResolvedValue( + context({ + campaignEnrollmentReason: 'install_runtime', + enrichmentArtifact: null, + }) + ), + sendRecipient: vi.fn().mockResolvedValue({ accepted: false, reason }), + }); + await expect( + dispatchLifecycleAppOwnedJob( + {} as SqlExecutor, + job('send_step', { campaign_version: 'v1', step: 1 }), + {}, + deps + ) + ).resolves.toBe('deferred'); + expect(deps.deferJob).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ errorCode: reason }) + ); + } + ); + it('fulfills the persisted form request through the recipient boundary', async () => { const deps = dependencies(); const fulfill = job('fulfill', { @@ -766,14 +903,35 @@ describe('loadLifecycleRuntimeConfiguration', () => { }); }); - it('defaults all three delivery switches off', () => { + it('defaults delivery and install-runtime activation switches off', () => { expect(loadLifecycleRuntimeConfiguration({})).toMatchObject({ campaignEnrollmentEnabled: false, campaignEnabled: false, deliveryEnabled: false, + installRuntimeHelloEnabled: false, }); }); + it('enables install-runtime hello only with the exact configured boolean', () => { + expect( + loadLifecycleRuntimeConfiguration({ + GROWTH_INSTALL_RUNTIME_HELLO_ENABLED: 'true', + }) + ).toMatchObject({ installRuntimeHelloEnabled: true }); + expect( + loadLifecycleRuntimeConfiguration({ + GROWTH_INSTALL_RUNTIME_HELLO_ENABLED: 'false', + }) + ).toMatchObject({ installRuntimeHelloEnabled: false }); + for (const value of ['TRUE', '1', ' true ', '']) { + expect(() => + loadLifecycleRuntimeConfiguration({ + GROWTH_INSTALL_RUNTIME_HELLO_ENABLED: value, + }) + ).toThrow(/GROWTH_INSTALL_RUNTIME_HELLO_ENABLED/); + } + }); + it('runs enrichment with every mail environment variable absent and delivery disabled', async () => { const deps = createDefaultLifecycleJobDependencies({ CAMPAIGN_ENROLLMENT_ENABLED: 'false', diff --git a/apps/lifecycle/src/campaign/send.ts b/apps/lifecycle/src/campaign/send.ts index 78c3d1794..33703a8b2 100644 --- a/apps/lifecycle/src/campaign/send.ts +++ b/apps/lifecycle/src/campaign/send.ts @@ -76,6 +76,7 @@ export interface LifecycleJobContext { emailClassification: 'work' | 'personal' | 'unknown'; formSubmission: Record; enrollmentAt: Date | null; + campaignEnrollmentReason?: 'install_runtime' | null; enrichmentArtifact: GrowthArtifact | null; } @@ -174,6 +175,7 @@ export interface LifecycleJobDependencies { export interface LifecycleRuntimeConfiguration { campaignEnrollmentEnabled: boolean; + installRuntimeHelloEnabled: boolean; campaignEnrollmentStartAt?: Date; campaignEnabled: boolean; deliveryEnabled: boolean; @@ -264,11 +266,12 @@ export function prepareCampaignMessage(input: { unsubscribeUrl: UnsubscribeActionUrl; }): PreparedCampaignMessage { const step = campaignStep(input.job); - const artifact = validArtifact( - input.context.enrichmentArtifact, - input.context.contactId - ); - if (step === 1 && !artifact) { + const genericHello = + input.context.campaignEnrollmentReason === 'install_runtime'; + const artifact = genericHello + ? null + : validArtifact(input.context.enrichmentArtifact, input.context.contactId); + if (step === 1 && !artifact && !genericHello) { if (!input.context.enrollmentAt) { throw new DeterministicLifecycleJobError( 'Campaign enrollment timestamp is required' @@ -662,6 +665,10 @@ export function loadLifecycleRuntimeConfiguration( 'CAMPAIGN_ENROLLMENT_ENABLED' ); const campaignEnabled = exactBoolean(environment, 'CAMPAIGN_ENABLED'); + const installRuntimeHelloEnabled = exactBoolean( + environment, + 'GROWTH_INSTALL_RUNTIME_HELLO_ENABLED' + ); const deliveryEnabled = exactBoolean(environment, 'DELIVERY_ENABLED'); let campaignEnrollmentStartAt: Date | undefined; if (campaignEnrollmentEnabled) { @@ -682,6 +689,7 @@ export function loadLifecycleRuntimeConfiguration( } return { campaignEnrollmentEnabled, + installRuntimeHelloEnabled, ...(campaignEnrollmentStartAt ? { campaignEnrollmentStartAt } : {}), campaignEnabled, deliveryEnabled, diff --git a/apps/lifecycle/src/dispatcher.spec.ts b/apps/lifecycle/src/dispatcher.spec.ts index 314961a4b..7486b7cab 100644 --- a/apps/lifecycle/src/dispatcher.spec.ts +++ b/apps/lifecycle/src/dispatcher.spec.ts @@ -21,6 +21,9 @@ import { } from './campaign/send.js'; const NOW = new Date('2026-09-01T12:00:00.000Z'); +const EMAIL_KEYRING = { + active: { version: 1, secret: 'dispatcher-email-test-secret-material' }, +}; afterEach(() => vi.useRealTimers()); @@ -61,6 +64,13 @@ function dependencies( dispatchLeasedJob: vi.fn().mockResolvedValue('completed'), isRecoveryPaused: vi.fn().mockResolvedValue(false), leaseDueJobs: vi.fn().mockResolvedValue([]), + loadEmailKeyring: vi.fn(() => EMAIL_KEYRING), + processInstallRuntimeActivations: vi.fn().mockResolvedValue({ + approved: 0, + ineligible: 0, + conflicted: 0, + disabled: false, + }), materializeCampaignEnrollment: vi.fn().mockResolvedValue({ enrolledContactIds: [], createdJobs: 0, @@ -280,6 +290,7 @@ describe('dispatchLifecycleJobs', () => { batchSize: 10, campaignEnabled: false, campaignEnrollmentEnabled: true, + installRuntimeHelloEnabled: true, campaignEnrollmentStartAt: start, signal: new AbortController().signal, }, @@ -295,6 +306,17 @@ describe('dispatchLifecycleJobs', () => { batchSize: 10, } ); + expect(deps.loadEmailKeyring).toHaveBeenCalledOnce(); + expect(deps.processInstallRuntimeActivations).toHaveBeenCalledWith( + expect.anything(), + { enabled: true, limit: 10, now: NOW, keyring: EMAIL_KEYRING } + ); + expect( + vi.mocked(deps.processInstallRuntimeActivations).mock + .invocationCallOrder[0] + ).toBeLessThan( + materializeCampaignEnrollment.mock.invocationCallOrder[0] ?? 0 + ); expect( materializeCampaignEnrollment.mock.invocationCallOrder[0] ).toBeLessThan( @@ -302,6 +324,34 @@ describe('dispatchLifecycleJobs', () => { ); }); + it.each([undefined, false])( + 'keeps form and claim enrollment working without new keys when hello rollout is %s', + async (installRuntimeHelloEnabled) => { + const deps = dependencies({ + loadEmailKeyring: vi.fn(() => { + throw new Error('new HMAC keys are not configured'); + }), + }); + await expect( + dispatchLifecycleJobs( + { + batchSize: 10, + campaignEnabled: true, + campaignEnrollmentEnabled: true, + installRuntimeHelloEnabled, + campaignEnrollmentStartAt: NOW, + signal: new AbortController().signal, + }, + deps + ) + ).resolves.toMatchObject({ leased: 0 }); + expect(deps.materializeCampaignEnrollment).toHaveBeenCalledOnce(); + expect(deps.leaseDueJobs).toHaveBeenCalledOnce(); + expect(deps.loadEmailKeyring).not.toHaveBeenCalled(); + expect(deps.processInstallRuntimeActivations).not.toHaveBeenCalled(); + } + ); + it('does no enrollment work when enrollment is disabled', async () => { const deps = dependencies(); @@ -310,12 +360,41 @@ describe('dispatchLifecycleJobs', () => { batchSize: 10, campaignEnabled: true, campaignEnrollmentEnabled: false, + installRuntimeHelloEnabled: true, signal: new AbortController().signal, }, deps ); expect(deps.materializeCampaignEnrollment).not.toHaveBeenCalled(); + expect(deps.processInstallRuntimeActivations).not.toHaveBeenCalled(); + expect(deps.loadEmailKeyring).not.toHaveBeenCalled(); + }); + + it('stops before enrollment and leasing if activation processing is cancelled', async () => { + const controller = new AbortController(); + const deps = dependencies({ + processInstallRuntimeActivations: vi.fn().mockImplementation(async () => { + controller.abort(new Error('activation cancelled')); + return { approved: 0, ineligible: 0, conflicted: 0, disabled: false }; + }), + }); + await expect( + dispatchLifecycleJobs( + { + batchSize: 10, + campaignEnabled: true, + campaignEnrollmentEnabled: true, + installRuntimeHelloEnabled: true, + campaignEnrollmentStartAt: NOW, + signal: controller.signal, + }, + deps + ) + ).rejects.toThrow('activation cancelled'); + expect(deps.materializeCampaignEnrollment).not.toHaveBeenCalled(); + expect(deps.leaseDueJobs).not.toHaveBeenCalled(); + expect(deps.createDatabase().close).toHaveBeenCalledOnce(); }); it.each([0, 26, 1.5])( diff --git a/apps/lifecycle/src/dispatcher.ts b/apps/lifecycle/src/dispatcher.ts index 5ed5cc5c1..0aec20a74 100644 --- a/apps/lifecycle/src/dispatcher.ts +++ b/apps/lifecycle/src/dispatcher.ts @@ -5,6 +5,7 @@ import { isGoogleMailboxRecoveryPaused, leaseDueJobs, materializeCampaignEnrollment, + processInstallRuntimeActivations, renewJobLease, type GrowthAppJobHandlers, type GrowthDispatchDependencies, @@ -15,6 +16,7 @@ import { import { createLifecycleAppJobHandlers } from './campaign/send.js'; import { DeterministicLifecycleJobError } from './job-errors.js'; +import { loadEmailHmacKeyring } from './email-keyring.js'; export { DeterministicLifecycleJobError } from './job-errors.js'; @@ -33,6 +35,7 @@ export interface LifecycleDispatcherInput { batchSize: number; campaignEnabled: boolean; campaignEnrollmentEnabled?: boolean; + installRuntimeHelloEnabled?: boolean; campaignEnrollmentStartAt?: Date; signal: AbortSignal; } @@ -55,6 +58,8 @@ export interface LifecycleDispatcherDependencies { isRecoveryPaused: typeof isGoogleMailboxRecoveryPaused; leaseDueJobs: typeof leaseDueJobs; materializeCampaignEnrollment: typeof materializeCampaignEnrollment; + processInstallRuntimeActivations: typeof processInstallRuntimeActivations; + loadEmailKeyring: typeof loadEmailHmacKeyring; now: () => Date; renewJobLease: typeof renewJobLease; quarantineJob: typeof failLeasedJob; @@ -69,6 +74,8 @@ const defaultDependencies: LifecycleDispatcherDependencies = { isRecoveryPaused: isGoogleMailboxRecoveryPaused, leaseDueJobs, materializeCampaignEnrollment, + processInstallRuntimeActivations, + loadEmailKeyring: loadEmailHmacKeyring, now: () => new Date(), renewJobLease, quarantineJob: failLeasedJob, @@ -158,6 +165,15 @@ export async function dispatchLifecycleJobs( 'campaignEnrollmentStartAt is required when enrollment is enabled' ); } + if (input.installRuntimeHelloEnabled === true) { + await dependencies.processInstallRuntimeActivations(executor, { + enabled: true, + limit: batchSize, + now: dependencies.now(), + keyring: dependencies.loadEmailKeyring(), + }); + input.signal.throwIfAborted(); + } await dependencies.materializeCampaignEnrollment(executor, { enrollmentEnabled: true, enrollmentStartAt: input.campaignEnrollmentStartAt, diff --git a/apps/lifecycle/src/email-keyring.spec.ts b/apps/lifecycle/src/email-keyring.spec.ts new file mode 100644 index 000000000..3805c1b99 --- /dev/null +++ b/apps/lifecycle/src/email-keyring.spec.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest'; +import { loadEmailHmacKeyring } from './email-keyring.js'; + +const active = { version: 2, secret: 'a'.repeat(32) }; +const environment = { + GROWTH_EMAIL_HMAC_ACTIVE_VERSION: '2', + GROWTH_EMAIL_HMAC_ACTIVE_SECRET: active.secret, +}; + +describe('lifecycle email HMAC configuration', () => { + it('loads the same active and rotation keys used by collection', () => { + const previous = [{ version: 1, secret: 'b'.repeat(32) }]; + expect( + loadEmailHmacKeyring({ + ...environment, + GROWTH_EMAIL_HMAC_PREVIOUS_KEYS: JSON.stringify(previous), + }) + ).toEqual({ active, previous }); + expect(loadEmailHmacKeyring(environment)).toEqual({ active }); + }); + + it('requires an adequately sized active key when configuration is read', () => { + expect(() => loadEmailHmacKeyring({})).toThrow(/active secret/); + expect(() => + loadEmailHmacKeyring({ + ...environment, + GROWTH_EMAIL_HMAC_ACTIVE_SECRET: 'short', + }) + ).toThrow(/active secret/); + }); + + it.each(['0', '32768', '1.5', 'NaN'])( + 'rejects an invalid active key version %s', + (version) => { + expect(() => + loadEmailHmacKeyring({ + ...environment, + GROWTH_EMAIL_HMAC_ACTIVE_VERSION: version, + }) + ).toThrow(/active version/); + } + ); + + it('rejects duplicate key versions', () => { + expect(() => + loadEmailHmacKeyring({ + ...environment, + GROWTH_EMAIL_HMAC_PREVIOUS_KEYS: JSON.stringify([active]), + }) + ).toThrow(/unique/); + }); + + it('does not expose malformed previous key material in errors', () => { + const secret = 'private-malformed-key-material'; + let caught: unknown; + try { + loadEmailHmacKeyring({ + ...environment, + GROWTH_EMAIL_HMAC_PREVIOUS_KEYS: secret, + }); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect(String(caught)).not.toContain(secret); + expect(String(caught)).toContain('previous keys are invalid'); + }); +}); diff --git a/apps/lifecycle/src/email-keyring.ts b/apps/lifecycle/src/email-keyring.ts new file mode 100644 index 000000000..859c9e570 --- /dev/null +++ b/apps/lifecycle/src/email-keyring.ts @@ -0,0 +1,67 @@ +import type { EmailHmacKey, EmailHmacKeyring } from './growth.js'; + +function version(value: string | undefined): number { + if (!value || !/^\d+$/u.test(value)) { + throw new Error('Growth email HMAC active version is required'); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 32_767) { + throw new Error('Growth email HMAC active version is invalid'); + } + return parsed; +} + +function previousKey(candidate: unknown): EmailHmacKey { + if ( + candidate === null || + typeof candidate !== 'object' || + Array.isArray(candidate) + ) { + throw new Error('Growth email HMAC previous key is invalid'); + } + const record = candidate as Record; + if ( + Object.keys(record).length !== 2 || + typeof record['version'] !== 'number' || + !Number.isSafeInteger(record['version']) || + record['version'] < 1 || + record['version'] > 32_767 || + typeof record['secret'] !== 'string' || + Buffer.byteLength(record['secret'], 'utf8') < 32 + ) + throw new Error('Growth email HMAC previous key is invalid'); + return { version: record['version'], secret: record['secret'] }; +} + +// Keep lifecycle's server-only configuration aligned with collection's keyring. +// Load lazily: disabled activation must not require identity credentials. +export function loadEmailHmacKeyring( + environment: Readonly> = process.env +): EmailHmacKeyring { + const secret = environment['GROWTH_EMAIL_HMAC_ACTIVE_SECRET']; + if (!secret || Buffer.byteLength(secret, 'utf8') < 32) { + throw new Error('Growth email HMAC active secret is required'); + } + const active = { + version: version(environment['GROWTH_EMAIL_HMAC_ACTIVE_VERSION']), + secret, + }; + const rawPrevious = environment['GROWTH_EMAIL_HMAC_PREVIOUS_KEYS']; + if (!rawPrevious) return { active }; + let parsed: unknown; + try { + parsed = JSON.parse(rawPrevious) as unknown; + } catch { + throw new Error('Growth email HMAC previous keys are invalid'); + } + if (!Array.isArray(parsed)) + throw new Error('Growth email HMAC previous keys must be an array'); + const previous = parsed.map(previousKey); + const versions = new Set([active.version]); + for (const key of previous) { + if (versions.has(key.version)) + throw new Error('Growth email HMAC key versions must be unique'); + versions.add(key.version); + } + return { active, previous }; +} diff --git a/apps/website/next.config.ts b/apps/website/next.config.ts index a667249c9..0a349ac9a 100644 --- a/apps/website/next.config.ts +++ b/apps/website/next.config.ts @@ -22,6 +22,8 @@ export const nextConfig: WithNxOptions = { // route deploys with no corpus and returns empty for every query — // silently, and only in production. 'content/docs/**/*.mdx', + // Growth validates blog observations against the catalog at request time. + 'content/blog/**/*.mdx', ], }, skipTrailingSlashRedirect: true, diff --git a/apps/website/src/app/api/growth/collect/v1/[source]/route.spec.ts b/apps/website/src/app/api/growth/collect/v1/[source]/route.spec.ts new file mode 100644 index 000000000..f5e036fc2 --- /dev/null +++ b/apps/website/src/app/api/growth/collect/v1/[source]/route.spec.ts @@ -0,0 +1,29 @@ +import { vi } from 'vitest'; +const mocks = vi.hoisted(() => ({ + handle: vi.fn(async () => new Response(null, { status: 204 })), + defaults: { marker: 'dependencies' }, +})); +vi.mock('../../../../../../lib/growth/collection-route', () => ({ + defaultCollectionRouteDependencies: () => mocks.defaults, + createCollectionRoute: vi.fn(() => mocks.handle), +})); +import { POST, OPTIONS, runtime } from './route'; +import { createCollectionRoute } from '../../../../../../lib/growth/collection-route'; +it('wires POST and OPTIONS through the shared Node collection handler', async () => { + expect(runtime).toBe('nodejs'); + expect(createCollectionRoute).toHaveBeenCalledWith(mocks.defaults); + for (const [method, handler] of [ + ['POST', POST], + ['OPTIONS', OPTIONS], + ] as const) { + const request = new Request('https://example.invalid', { method }); + expect( + ( + await handler(request, { + params: Promise.resolve({ source: 'runtime' }), + }) + ).status + ).toBe(204); + expect(mocks.handle).toHaveBeenLastCalledWith(request, 'runtime'); + } +}); diff --git a/apps/website/src/app/api/growth/collect/v1/[source]/route.ts b/apps/website/src/app/api/growth/collect/v1/[source]/route.ts new file mode 100644 index 000000000..993767e7d --- /dev/null +++ b/apps/website/src/app/api/growth/collect/v1/[source]/route.ts @@ -0,0 +1,14 @@ +import { + createCollectionRoute, + defaultCollectionRouteDependencies, +} from '../../../../../../lib/growth/collection-route'; + +export const runtime = 'nodejs'; +const handle = createCollectionRoute(defaultCollectionRouteDependencies()); +type Context = { params: Promise<{ source: string }> }; +export async function POST(request: Request, context: Context) { + return handle(request, (await context.params).source); +} +export async function OPTIONS(request: Request, context: Context) { + return handle(request, (await context.params).source); +} diff --git a/apps/website/src/lib/growth/collection-route.spec.ts b/apps/website/src/lib/growth/collection-route.spec.ts new file mode 100644 index 000000000..dd26fc0ac --- /dev/null +++ b/apps/website/src/lib/growth/collection-route.spec.ts @@ -0,0 +1,401 @@ +import { vi } from 'vitest'; +// Growth's migration tests create a project-graph cycle through scripts, not a runtime dependency. +// eslint-disable-next-line @nx/enforce-module-boundaries +import { ObservationError } from '@threadplane-internal/growth'; +vi.mock('server-only', () => ({})); +import { + createCollectionRoute, + validateWebsiteCollection, + type CollectionRouteDependencies, +} from './collection-route'; +import { selectRuntimeAnnouncements } from './runtime-announcements'; + +function setup() { + const close = vi.fn(async () => undefined); + const deps = { + environment: () => ({ GROWTH_COLLECTION_SOURCES: 'website' }), + createDatabase: vi.fn(() => ({ close })), + loadKeyring: () => ({ active: { version: 1, secret: 'x'.repeat(32) } }), + now: () => new Date('2026-09-04T12:00:00Z'), + sourceBudget: vi.fn(async () => ({ allowed: true, retryAfterSec: 30 })), + subjectBudgets: vi.fn(async () => ({ allowed: true, retryAfterSec: 30 })), + accept: vi.fn(async () => ({ + schemaVersion: 1, + events: [ + { + eventId: '11111111-1111-4111-8111-111111111111', + disposition: 'accepted', + }, + ], + })), + log: vi.fn(), + validateWebsite: validateWebsiteCollection, + runtimeAnnouncements: vi.fn< + CollectionRouteDependencies['runtimeAnnouncements'] + >(() => [announcement]), + }; + return { + deps, + handle: createCollectionRoute( + deps as unknown as CollectionRouteDependencies + ), + close, + }; +} +const batch = { + schemaVersion: 1, + events: [ + { + eventId: '11111111-1111-4111-8111-111111111111', + kind: 'website.session_started', + occurredAt: '2026-09-04T12:00:00Z', + collectorVersion: '1', + subject: { + id: '22222222-2222-4222-8222-222222222222', + namespace: 'website_session', + scope: 'session', + }, + properties: {}, + }, + ], +}; +const request = (value: unknown = batch) => + new Request('https://example.invalid/api/growth/collect/v1/website', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(value), + }); +const announcement = { + id: 'runtime-docs-v1', + packageNames: ['@threadplane/langgraph'], + minVersion: '0.0.0', + expiresAt: '2027-09-04T00:00:00Z', + text: 'Explore the Threadplane documentation.', + documentationUrl: 'https://threadplane.ai/docs', +}; +const runtimeBatch = { + schemaVersion: 1, + events: [ + { + ...batch.events[0], + kind: 'runtime.session_started', + occurredAt: '2026-09-04T12:00:00.000Z', + subject: { ...batch.events[0].subject, namespace: 'development_browser' }, + sessionId: '33333333-3333-4333-8333-333333333333', + properties: { + packageName: '@threadplane/langgraph', + packageVersion: '0.0.65', + integration: 'langgraph', + }, + }, + ], +}; +describe('runtime announcement exchange', () => { + it.each(['0.0.65', 'unknown', '0.0.65-beta.1'])( + 'selects the public catalog using committed batch version %s', + async (packageVersion) => { + const { deps, handle } = setup(); + deps.environment = () => ({ GROWTH_COLLECTION_SOURCES: 'runtime' }); + deps.runtimeAnnouncements.mockImplementation(selectRuntimeAnnouncements); + const event = runtimeBatch.events[0]; + const response = await handle( + request({ + ...runtimeBatch, + events: [ + { ...event, properties: { ...event.properties, packageVersion } }, + ], + }), + 'runtime' + ); + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.events).toEqual( + (await deps.accept.mock.results[0].value).events + ); + expect(body.announcements).toHaveLength( + packageVersion === '0.0.65' ? 1 : 0 + ); + } + ); + it('adds applicable announcements only after durable acceptance resolves', async () => { + const { deps, handle } = setup(); + deps.environment = () => ({ GROWTH_COLLECTION_SOURCES: 'runtime' }); + let commit!: () => void; + const acknowledgment = { + schemaVersion: 1, + events: [{ eventId: batch.events[0].eventId, disposition: 'accepted' }], + }; + deps.accept.mockImplementationOnce( + () => + new Promise((resolve) => { + commit = () => resolve(acknowledgment); + }) + ); + const pending = handle(request(runtimeBatch), 'runtime'); + await vi.waitFor(() => expect(deps.accept).toHaveBeenCalledOnce()); + expect(deps.runtimeAnnouncements).not.toHaveBeenCalled(); + commit(); + const response = await pending; + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + ...acknowledgment, + announcements: [announcement], + }); + expect(deps.runtimeAnnouncements).toHaveBeenCalledWith( + runtimeBatch, + deps.now() + ); + }); + it('preserves committed acknowledgment when catalog selection fails', async () => { + const { deps, handle } = setup(); + deps.environment = () => ({ GROWTH_COLLECTION_SOURCES: 'runtime' }); + deps.runtimeAnnouncements.mockImplementation(() => { + throw new Error('catalog unavailable'); + }); + const response = await handle(request(runtimeBatch), 'runtime'); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + ...(await deps.accept.mock.results[0].value), + announcements: [], + }); + }); + it('keeps website acknowledgments unchanged', async () => { + const { deps, handle } = setup(); + const response = await handle(request(), 'website'); + expect(await response.json()).toEqual( + await deps.accept.mock.results[0].value + ); + expect(deps.runtimeAnnouncements).not.toHaveBeenCalled(); + }); + it.each(['disabled', 'invalid', 'failed acceptance'] as const)( + 'returns no announcements on %s', + async (failure) => { + const { deps, handle } = setup(); + deps.environment = () => ({ + GROWTH_COLLECTION_SOURCES: failure === 'disabled' ? '' : 'runtime', + }); + if (failure === 'failed acceptance') + deps.accept.mockRejectedValueOnce(new Error('database unavailable')); + const response = await handle( + request(failure === 'invalid' ? {} : runtimeBatch), + 'runtime' + ); + expect(response.status).toBe(failure === 'invalid' ? 400 : 503); + expect(await response.json()).not.toHaveProperty('announcements'); + expect(deps.runtimeAnnouncements).not.toHaveBeenCalled(); + } + ); + it('provides credential-free preflight and browser-visible Retry-After', async () => { + const { deps, handle } = setup(); + const preflight = await handle( + new Request('https://example.invalid', { method: 'OPTIONS' }), + 'runtime' + ); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + expect(preflight.headers.get('access-control-allow-methods')).toBe( + 'POST, OPTIONS' + ); + expect( + preflight.headers.get('access-control-allow-credentials') + ).toBeNull(); + expect(deps.createDatabase).not.toHaveBeenCalled(); + const disabled = await handle(request(runtimeBatch), 'runtime'); + expect(disabled.headers.get('retry-after')).toBe('60'); + expect(disabled.headers.get('access-control-expose-headers')).toBe( + 'Retry-After' + ); + }); +}); +describe('collection HTTP adapter', () => { + it('rejects forged content and campaign metadata before durable acceptance', async () => { + const { handle, deps } = setup(); + for (const properties of [ + { contentId: 'unknown-private-page', topic: 'other' }, + { contentId: 'home', topic: 'pricing' }, + { contentId: 'https://private.invalid?q=secret', topic: 'other' }, + ]) { + expect( + ( + await handle( + request({ + ...batch, + events: [ + { + ...batch.events[0], + kind: 'website.content_viewed', + properties, + }, + ], + }), + 'website' + ) + ).status + ).toBe(400); + } + for (const campaignSource of [ + 'reader@example.invalid', + 'https://private.invalid', + 'private?query=secret', + ]) + expect( + ( + await handle( + request({ + ...batch, + events: [{ ...batch.events[0], properties: { campaignSource } }], + }), + 'website' + ) + ).status + ).toBe(400); + expect(deps.accept).not.toHaveBeenCalled(); + expect( + ( + await handle( + request({ + ...batch, + events: [ + { + ...batch.events[0], + kind: 'website.content_viewed', + properties: { contentId: 'home', topic: 'getting_started' }, + }, + ], + }), + 'website' + ) + ).status + ).toBe(200); + }); + it('cancels a stalled request body within its deadline', async () => { + const { handle, deps } = setup(); + const cancel = vi.fn(); + const body = new ReadableStream({ cancel }); + const response = await handle( + new Request('https://example.invalid', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body, + duplex: 'half', + } as RequestInit), + 'website' + ); + expect(response.status).toBe(413); + await vi.waitFor(() => expect(cancel).toHaveBeenCalledOnce()); + expect(deps.accept).not.toHaveBeenCalled(); + }); + it('maps malformed JSON, content type, and atomic conflicts without leaking values', async () => { + const { handle, deps } = setup(); + expect( + ( + await handle( + new Request('https://example.invalid', { + method: 'POST', + body: '{', + headers: { 'content-type': 'application/json' }, + }), + 'website' + ) + ).status + ).toBe(400); + expect( + ( + await handle( + new Request('https://example.invalid', { + method: 'POST', + body: '{}', + }), + 'website' + ) + ).status + ).toBe(415); + deps.accept.mockRejectedValueOnce(new ObservationError('event_conflict')); + expect((await handle(request(), 'website')).status).toBe(409); + }); + it('accepts identity-free website activity without an email keyring', async () => { + const { deps, handle } = setup(); + deps.loadKeyring = () => { + throw new Error('key unavailable'); + }; + expect((await handle(request(), 'website')).status).toBe(200); + }); + it('treats an invalid server source configuration as unavailable', async () => { + const { deps, handle } = setup(); + deps.environment = () => ({ GROWTH_COLLECTION_SOURCES: 'misspelled' }); + expect((await handle(request(), 'website')).status).toBe(503); + expect(deps.createDatabase).not.toHaveBeenCalled(); + }); + it('does not acknowledge before the durable operation completes', async () => { + const { deps, handle, close } = setup(); + let commit!: () => void; + deps.accept.mockImplementationOnce( + () => + new Promise((resolve) => { + commit = () => resolve({ schemaVersion: 1, events: [] }); + }) + ); + const pending = handle(request(), 'website'); + let returned = false; + pending.then(() => { + returned = true; + }); + await vi.waitFor(() => expect(deps.accept).toHaveBeenCalledOnce()); + expect(returned).toBe(false); + commit(); + const response = await pending; + expect(response.status).toBe(200); + expect(response.headers.get('access-control-allow-origin')).toBe('*'); + expect(close).toHaveBeenCalledOnce(); + }); + it('fails closed when disabled and handles preflight without a database', async () => { + const { deps, handle } = setup(); + deps.environment = () => ({ GROWTH_COLLECTION_SOURCES: '' }); + expect((await handle(request(), 'website')).status).toBe(503); + expect( + ( + await handle( + new Request('https://example.invalid', { method: 'OPTIONS' }), + 'website' + ) + ).status + ).toBe(204); + expect(deps.createDatabase).not.toHaveBeenCalled(); + }); + it('rejects unknown trust fields and charges source budget for invalid input', async () => { + const { deps, handle } = setup(); + expect( + (await handle(request({ ...batch, trust: 'server_verified' }), 'website')) + .status + ).toBe(400); + expect(deps.sourceBudget).toHaveBeenCalledOnce(); + expect(deps.accept).not.toHaveBeenCalled(); + }); + it('does not reveal storage errors or private payloads', async () => { + const { deps, handle } = setup(); + deps.accept.mockRejectedValueOnce(new Error('DO-NOT-LOG@example.invalid')); + const response = await handle(request(), 'website'); + expect(response.status).toBe(503); + expect(await response.text()).not.toContain('DO-NOT-LOG'); + expect(JSON.stringify(deps.log.mock.calls)).not.toContain('DO-NOT-LOG'); + }); + it('reports quotas and closes the database', async () => { + const { deps, handle, close } = setup(); + deps.sourceBudget.mockResolvedValueOnce({ + allowed: false, + retryAfterSec: 30, + }); + const response = await handle(request(), 'website'); + expect(response.status).toBe(429); + expect(response.headers.get('retry-after')).toBe('30'); + expect(close).toHaveBeenCalledOnce(); + }); + it('rejects oversized bodies and unknown sources', async () => { + const { handle, deps } = setup(); + expect( + (await handle(request({ data: 'x'.repeat(65536) }), 'website')).status + ).toBe(413); + expect((await handle(request(), 'attacker')).status).toBe(404); + expect(deps.accept).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/website/src/lib/growth/collection-route.ts b/apps/website/src/lib/growth/collection-route.ts new file mode 100644 index 000000000..40ca48679 --- /dev/null +++ b/apps/website/src/lib/growth/collection-route.ts @@ -0,0 +1,224 @@ +import 'server-only'; +// eslint-disable-next-line @nx/enforce-module-boundaries +import { + acceptObservationBatch, + consumeSourceBudget, + consumeSubjectBudgets, + createDatabaseExecutor, + parseCollectionBatch, + collectionSource, + ObservationError, + MAX_BODY_BYTES, + type CollectionBatchV1, + type SqlExecutor, + type EmailHmacKeyring, +} from '@threadplane-internal/growth'; +import { readBoundedBody } from '../../app/api/_internal/read-bounded-body'; +import { loadEmailHmacKeyring } from './email-keyring'; +import { isKnownWebsiteContent } from './website-content'; +import { selectRuntimeAnnouncements } from './runtime-announcements'; + +export interface CollectionRouteDependencies { + environment(): Readonly>; + createDatabase(): SqlExecutor; + loadKeyring(): EmailHmacKeyring; + now(): Date; + sourceBudget: typeof consumeSourceBudget; + subjectBudgets: typeof consumeSubjectBudgets; + accept: typeof acceptObservationBatch; + validateWebsite(batch: CollectionBatchV1): void; + runtimeAnnouncements: typeof selectRuntimeAnnouncements; + log(event: Readonly>): void; +} +export function validateWebsiteCollection(batch: CollectionBatchV1): void { + for (const event of batch.events) + if ( + event.kind === 'website.content_viewed' && + !isKnownWebsiteContent(event.properties.contentId, event.properties.topic) + ) + throw new ObservationError('invalid_payload'); +} +const headers = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type', + 'Access-Control-Expose-Headers': 'Retry-After', + 'Cache-Control': 'no-store', +}; +function response( + status: number, + body: unknown, + retryAfter?: number +): Response { + return new Response(body === null ? null : JSON.stringify(body), { + status, + headers: { + ...headers, + 'Content-Type': 'application/json', + ...(retryAfter === undefined + ? {} + : { 'Retry-After': String(retryAfter) }), + }, + }); +} +async function readCollectionBody(request: Request): Promise { + let timeout: ReturnType | undefined; + // Cancel the reader's underlying stream on timeout rather than leave a hung body read alive. + const controller = new AbortController(); + const relay = new TransformStream(); + const piping = request.body + ?.pipeTo(relay.writable, { signal: controller.signal }) + .catch(() => undefined); + try { + const boundedRequest = new Request(request.url, { + method: 'POST', + headers: request.headers, + body: request.body ? relay.readable : null, + duplex: 'half', + } as RequestInit); + return await Promise.race([ + readBoundedBody(boundedRequest, MAX_BODY_BYTES), + new Promise((resolve) => { + timeout = setTimeout(() => { + controller.abort(); + resolve(null); + }, 3000); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + controller.abort(); + void piping; + } +} +export function createCollectionRoute(deps: CollectionRouteDependencies) { + return async (request: Request, sourceInput: unknown): Promise => { + let source; + try { + source = collectionSource(sourceInput); + } catch { + return response(404, { error: 'unknown_source' }); + } + if (request.method === 'OPTIONS') return response(204, null); + if (request.method !== 'POST') + return response(405, { error: 'method_not_allowed' }); + const start = Date.now(); + let db: SqlExecutor | undefined; + let eventCount = 0; + const report = (status: number, code: string, retryAfter?: number) => { + try { + deps.log({ + event: 'growth.collection', + source, + code, + status, + eventCount, + latencyMs: Date.now() - start, + }); + } catch { + /* Diagnostics cannot change acceptance. */ + } + return response(status, { error: code }, retryAfter); + }; + try { + let configured; + try { + configured = (deps.environment()['GROWTH_COLLECTION_SOURCES'] ?? '') + .split(',') + .map((v) => v.trim()) + .filter(Boolean) + .map(collectionSource); + } catch { + return report(503, 'collection_configuration', 60); + } + if (!configured.includes(source)) + return report(503, 'collection_disabled', 60); + db = deps.createDatabase(); + const admission = await deps.sourceBudget(db, source, deps.now()); + if (!admission.allowed) + return report(429, 'rate_limited', admission.retryAfterSec); + if ( + request.headers + .get('content-type') + ?.split(';')[0] + .trim() + .toLowerCase() !== 'application/json' + ) + return report(415, 'content_type'); + const raw = await readCollectionBody(request); + if (raw === null) return report(413, 'body_limit'); + let json: unknown; + try { + json = JSON.parse(raw); + } catch { + return report(400, 'invalid_payload'); + } + const now = deps.now(); + const batch = parseCollectionBatch(source, json, now); + if (source === 'website') deps.validateWebsite(batch); + eventCount = batch.events.length; + const subjects = await deps.subjectBudgets(db, source, batch.events, now); + if (!subjects.allowed) + return report(429, 'rate_limited', subjects.retryAfterSec); + const result = await deps.accept(db, source, batch, { + now, + ...(batch.events.some((e) => e.identity) + ? { keyring: deps.loadKeyring() } + : {}), + }); + try { + deps.log({ + event: 'growth.collection', + source, + code: 'committed', + status: 200, + eventCount, + accepted: result.events.filter((e) => e.disposition === 'accepted') + .length, + duplicate: result.events.filter((e) => e.disposition === 'duplicate') + .length, + redacted: result.events.filter((e) => e.disposition === 'redacted') + .length, + latencyMs: Date.now() - start, + }); + } catch { + /* Committed result remains authoritative. */ + } + if (source === 'runtime') { + try { + return response(200, { + ...result, + announcements: deps.runtimeAnnouncements(batch, now), + }); + } catch { + // Catalog selection or serialization cannot invalidate a committed acknowledgment. + return response(200, { ...result, announcements: [] }); + } + } + return response(200, result); + } catch (error) { + if (error instanceof ObservationError) { + if (error.code === 'event_conflict') return report(409, error.code); + if (['invalid_payload', 'unsupported_version'].includes(error.code)) + return report(400, error.code); + } + return report(503, 'collection_unavailable', 30); + } finally { + await db?.close?.().catch(() => undefined); + } + }; +} +export function defaultCollectionRouteDependencies(): CollectionRouteDependencies { + return { + environment: () => process.env, + createDatabase: () => createDatabaseExecutor(), + loadKeyring: () => loadEmailHmacKeyring(), + now: () => new Date(), + sourceBudget: consumeSourceBudget, + subjectBudgets: consumeSubjectBudgets, + accept: acceptObservationBatch, + validateWebsite: validateWebsiteCollection, + runtimeAnnouncements: selectRuntimeAnnouncements, + log: (event) => console.info(JSON.stringify(event)), + }; +} diff --git a/apps/website/src/lib/growth/runtime-announcements.spec.ts b/apps/website/src/lib/growth/runtime-announcements.spec.ts new file mode 100644 index 000000000..792170b35 --- /dev/null +++ b/apps/website/src/lib/growth/runtime-announcements.spec.ts @@ -0,0 +1,132 @@ +import { vi } from 'vitest'; +vi.mock('server-only', () => ({})); +import { + selectRuntimeAnnouncements, + type RuntimeAnnouncement, +} from './runtime-announcements'; + +const now = new Date('2026-09-04T12:00:00Z'); +const announcement: RuntimeAnnouncement = { + id: 'docs-range-v1', + packageNames: ['@threadplane/langgraph'], + minVersion: '0.0.9', + maxVersion: '1.0.0', + expiresAt: '2027-09-04T00:00:00Z', + text: 'Explore the Threadplane documentation.', + documentationUrl: 'https://threadplane.ai/docs', +}; +const batch = ( + packageVersion = '0.0.65', + packageName = '@threadplane/langgraph' +) => ({ + events: [{ properties: { packageName, packageVersion } }], +}); + +describe('runtime announcement catalog', () => { + it.each([ + '@threadplane/chat', + '@threadplane/langgraph', + '@threadplane/ag-ui', + '@threadplane/render', + ])('invites supported package %s to the documentation', (packageName) => { + expect( + selectRuntimeAnnouncements(batch('0.0.65', packageName), now) + ).toEqual([ + expect.objectContaining({ + documentationUrl: 'https://threadplane.ai/docs', + }), + ]); + }); + it.each([ + ['0.0.8', false], + ['0.0.9', true], + ['0.0.10', true], + ['0.9.99', true], + ['1.0.0', false], + ['2.0.0', false], + ])( + 'applies numeric inclusive minimum and exclusive maximum for %s', + (version, expected) => { + expect( + selectRuntimeAnnouncements(batch(version), now, [announcement]) + ).toHaveLength(expected ? 1 : 0); + } + ); + it.each([ + 'unknown', + '', + '1', + 'v0.0.65', + '0.0.65-beta.1', + '0.0.65+build', + '00.0.65', + '0.0.9007199254740992', + ])('omits announcements for unknown or non-release version %s', (version) => { + expect( + selectRuntimeAnnouncements(batch(version), now, [announcement]) + ).toEqual([]); + }); + it('omits other packages and expired announcements including the exact expiry instant', () => { + expect( + selectRuntimeAnnouncements(batch('0.0.65', '@threadplane/render'), now, [ + announcement, + ]) + ).toEqual([]); + expect( + selectRuntimeAnnouncements(batch(), new Date(announcement.expiresAt), [ + announcement, + ]) + ).toEqual([]); + expect( + selectRuntimeAnnouncements(batch(), new Date('invalid'), [announcement]) + ).toEqual([]); + }); + it('selects a catalog entry once across matching events and caps responses at five', () => { + const catalog = Array.from({ length: 7 }, (_, index) => ({ + ...announcement, + id: `docs-${index}`, + })); + const result = selectRuntimeAnnouncements( + { events: [...batch().events, ...batch().events] }, + now, + catalog + ); + expect(result.map((item) => item.id)).toEqual([ + 'docs-0', + 'docs-1', + 'docs-2', + 'docs-3', + 'docs-4', + ]); + }); + it('copies only public fields without sharing mutable package arrays', () => { + const privateEntry = { + ...announcement, + internalNotes: 'private', + subject: 'private', + }; + const result = selectRuntimeAnnouncements(batch(), now, [privateEntry]); + expect(result).toEqual([announcement]); + expect(result[0].packageNames).not.toBe(privateEntry.packageNames); + }); + it.each([ + { text: 'x'.repeat(501) }, + { text: '' }, + { text: 'hello\u001b[31m' }, + { text: '' }, + { documentationUrl: 'http://threadplane.ai/docs' }, + { documentationUrl: 'https://threadplane.ai.evil.invalid/docs' }, + { documentationUrl: 'https://threadplane.ai/docs-elsewhere' }, + { documentationUrl: 'https://user:secret@threadplane.ai/docs' }, + { documentationUrl: 'https://threadplane.ai/docs?token=secret' }, + { minVersion: 'unknown' }, + { maxVersion: '0.0.9-beta.1' }, + { expiresAt: 'invalid' }, + ])('omits malformed or unsafe catalog entries: %j', (invalid) => { + expect( + selectRuntimeAnnouncements(batch(), now, [ + { ...announcement, ...invalid }, + ]) + ).toEqual([]); + }); +}); diff --git a/apps/website/src/lib/growth/runtime-announcements.ts b/apps/website/src/lib/growth/runtime-announcements.ts new file mode 100644 index 000000000..5776006d0 --- /dev/null +++ b/apps/website/src/lib/growth/runtime-announcements.ts @@ -0,0 +1,116 @@ +import 'server-only'; + +export interface RuntimeAnnouncement { + id: string; + packageNames: readonly string[]; + minVersion: string; + maxVersion?: string; + expiresAt: string; + text: string; + documentationUrl?: string; +} + +type RuntimePackageBatch = { + events: readonly { properties: Readonly> }[]; +}; + +const catalog: readonly RuntimeAnnouncement[] = [ + { + id: 'runtime-documentation-2026-09', + packageNames: [ + '@threadplane/chat', + '@threadplane/langgraph', + '@threadplane/ag-ui', + '@threadplane/render', + ], + minVersion: '0.0.0', + expiresAt: '2027-09-04T00:00:00Z', + text: 'Building with Threadplane? Explore the documentation for streaming conversations, durable threads, and generative UI.', + documentationUrl: 'https://threadplane.ai/docs', + }, +]; + +/** Only numeric release versions have a defined ordering in this small catalog. */ +function releaseVersion(value: string): number[] | undefined { + if (!/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(value)) + return undefined; + const parts = value.split('.').map(Number); + return parts.every(Number.isSafeInteger) ? parts : undefined; +} + +function compareVersion(left: number[], right: number[]): number { + for (let index = 0; index < 3; index++) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return 0; +} + +function approvedDocumentationUrl(value: string): boolean { + try { + const url = new URL(value); + return ( + url.protocol === 'https:' && + url.hostname === 'threadplane.ai' && + !url.port && + !url.username && + !url.password && + !url.search && + (url.pathname === '/docs' || url.pathname.startsWith('/docs/')) + ); + } catch { + return false; + } +} + +export function selectRuntimeAnnouncements( + batch: RuntimePackageBatch, + now: Date, + entries: readonly RuntimeAnnouncement[] = catalog +): RuntimeAnnouncement[] { + if (!Number.isFinite(now.getTime())) return []; + const selected: RuntimeAnnouncement[] = []; + for (const entry of entries) { + const minimum = releaseVersion(entry.minVersion); + const maximum = + entry.maxVersion === undefined + ? undefined + : releaseVersion(entry.maxVersion); + if ( + !minimum || + (entry.maxVersion !== undefined && !maximum) || + !(Date.parse(entry.expiresAt) > now.getTime()) || + !entry.text.trim() || + entry.text.length > 500 || + /[<>\p{Cc}]/u.test(entry.text) || + (entry.documentationUrl !== undefined && + !approvedDocumentationUrl(entry.documentationUrl)) + ) + continue; + const applies = batch.events.some(({ properties }) => { + const version = releaseVersion(properties.packageVersion); + return ( + version && + entry.packageNames.includes(properties.packageName) && + compareVersion(version, minimum) >= 0 && + (!maximum || compareVersion(version, maximum) < 0) + ); + }); + if (!applies || selected.some(({ id }) => id === entry.id)) continue; + // Explicit projection prevents internal catalog metadata from entering this public response. + selected.push({ + id: entry.id, + packageNames: [...entry.packageNames], + minVersion: entry.minVersion, + ...(entry.maxVersion === undefined + ? {} + : { maxVersion: entry.maxVersion }), + expiresAt: entry.expiresAt, + text: entry.text, + ...(entry.documentationUrl === undefined + ? {} + : { documentationUrl: entry.documentationUrl }), + }); + if (selected.length === 5) break; + } + return selected; +} diff --git a/apps/website/src/lib/growth/website-content.ts b/apps/website/src/lib/growth/website-content.ts new file mode 100644 index 000000000..5390e128c --- /dev/null +++ b/apps/website/src/lib/growth/website-content.ts @@ -0,0 +1,53 @@ +import 'server-only'; +import { getAllSlugs } from '../blog'; +import { docsConfig, specialDocsPages } from '../docs-config'; +import type { WebsiteCatalog, WebsiteTopic } from './website-metadata'; + +let cached: WebsiteCatalog | undefined; +export function websiteContentCatalog(): WebsiteCatalog { + if (cached) return cached; + const catalog: Record = + {}; + const add = (path: string, topic: WebsiteTopic) => { + const contentId = path === '/' ? 'home' : path.slice(1).toLowerCase(); + if (/^[a-z0-9][a-z0-9_/-]{0,119}$/u.test(contentId)) + catalog[path] = { contentId, topic }; + }; + for (const path of [ + '/', + '/docs', + '/contact', + '/blog', + '/langgraph', + '/chat', + '/render', + '/ag-ui', + ]) + add(path, 'getting_started'); + add('/pricing', 'pricing'); + add('/privacy', 'security'); + for (const library of docsConfig) + for (const section of library.sections) + for (const page of section.pages) { + add( + `/docs/${library.id}/${page.section}/${page.slug}`, + page.section === 'getting-started' + ? 'getting_started' + : page.slug.includes('deploy') + ? 'deployment' + : 'architecture' + ); + } + for (const page of specialDocsPages) add(page.path, 'comparison'); + for (const slug of getAllSlugs()) add(`/blog/${slug}`, 'architecture'); + cached = Object.freeze(catalog); + return cached; +} +export function isKnownWebsiteContent( + contentId: string, + topic: string +): boolean { + return Object.values(websiteContentCatalog()).some( + (content) => content.contentId === contentId && content.topic === topic + ); +} diff --git a/apps/website/src/lib/growth/website-metadata.spec.ts b/apps/website/src/lib/growth/website-metadata.spec.ts new file mode 100644 index 000000000..f8dec7dc4 --- /dev/null +++ b/apps/website/src/lib/growth/website-metadata.spec.ts @@ -0,0 +1,58 @@ +import { + acquisitionProperties, + installedPackages, + contentForPath, +} from './website-metadata'; + +describe('website metadata minimization', () => { + it('keeps campaign tokens and referrer host, excluding arbitrary URLs and query text', () => { + expect( + acquisitionProperties( + '?utm_source= Newsletter &utm_medium=email&utm_campaign=autumn-2026&search=private', + 'https://example.org/articles?secret=x#fragment' + ) + ).toEqual({ + campaignSource: 'newsletter', + campaignMedium: 'email', + campaignName: 'autumn-2026', + referrerHost: 'example.org', + }); + expect( + acquisitionProperties( + '?utm_source=reader%40example.org&utm_medium=https%3A%2F%2Fprivate.org&token=secret', + 'javascript:private' + ) + ).toEqual({}); + }); + it('emits only catalogued content IDs and ignores arbitrary paths', () => { + const catalog = { + '/docs/chat/quickstart': { + contentId: 'chat-quickstart', + topic: 'getting_started' as const, + }, + }; + expect(contentForPath('/docs/chat/quickstart', catalog)).toEqual( + catalog['/docs/chat/quickstart'] + ); + expect(contentForPath('/private/user@example.invalid', catalog)).toBeNull(); + expect( + contentForPath('/docs/chat/quickstart?secret=x', catalog) + ).toBeNull(); + expect(contentForPath('/docs/chat/quickstart#private', catalog)).toBeNull(); + }); + it('recognizes install commands without returning copied code or arbitrary package text', () => { + expect( + installedPackages( + 'npm install @threadplane/chat @threadplane/langgraph@latest rxjs' + ) + ).toEqual(['@threadplane/chat', '@threadplane/langgraph']); + expect(installedPackages('pnpm add @threadplane/render')).toEqual([ + '@threadplane/render', + ]); + expect( + installedPackages('import { chat } from "@threadplane/chat";') + ).toEqual([]); + expect(installedPackages('echo npm install @threadplane/chat')).toEqual([]); + expect(installedPackages('npm install @threadplane/chat-fake')).toEqual([]); + }); +}); diff --git a/apps/website/src/lib/growth/website-metadata.ts b/apps/website/src/lib/growth/website-metadata.ts new file mode 100644 index 000000000..6db5cb092 --- /dev/null +++ b/apps/website/src/lib/growth/website-metadata.ts @@ -0,0 +1,61 @@ +export type WebsiteTopic = + | 'getting_started' + | 'architecture' + | 'comparison' + | 'pricing' + | 'security' + | 'deployment' + | 'other'; +export type WebsiteContent = { contentId: string; topic: WebsiteTopic }; +export type WebsiteCatalog = Readonly>; +export function contentForPath( + pathname: string, + catalog: WebsiteCatalog +): WebsiteContent | null { + return Object.hasOwn(catalog, pathname) ? catalog[pathname] : null; +} +export function acquisitionProperties( + search: string, + referrer: string +): Record { + const properties: Record = {}; + const params = new URLSearchParams(search.slice(0, 4096)); + for (const [parameter, key] of [ + ['utm_source', 'campaignSource'], + ['utm_medium', 'campaignMedium'], + ['utm_campaign', 'campaignName'], + ]) { + const token = params.get(parameter)?.trim().toLowerCase(); + if (token && /^[a-z0-9][a-z0-9_-]{0,119}$/u.test(token)) + properties[key] = token; + } + try { + const url = new URL(referrer); + if ( + ['http:', 'https:'].includes(url.protocol) && + url.hostname.length <= 253 && + /^[a-z0-9.-]+$/u.test(url.hostname) + ) + properties.referrerHost = url.hostname; + } catch { + /* An absent or invalid referrer contributes no evidence. */ + } + return properties; +} +const PACKAGES = [ + '@threadplane/chat', + '@threadplane/langgraph', + '@threadplane/ag-ui', + '@threadplane/render', +]; +export function installedPackages(command: string): string[] { + if ( + command.length > 4096 || + !/^\s*(?:npm\s+(?:install|i)|(?:pnpm|yarn|bun)\s+add)\s/u.test(command) + ) + return []; + const tokens = command.trim().split(/\s+/u).slice(2); + return PACKAGES.filter((name) => + tokens.some((token) => token === name || token.startsWith(name + '@')) + ); +} diff --git a/libs/growth/project.json b/libs/growth/project.json index c81b3ab40..4bc0024b0 100644 --- a/libs/growth/project.json +++ b/libs/growth/project.json @@ -10,6 +10,7 @@ "{workspaceRoot}/scripts/apply-migrations*", "{workspaceRoot}/scripts/growth-database-preflight*", "{workspaceRoot}/scripts/growth-control*", + "{workspaceRoot}/scripts/growth-observability*", "{workspaceRoot}/scripts/import-resend-lifecycle*", "{workspaceRoot}/scripts/cancel-resend-lifecycle*" ] diff --git a/libs/growth/src/index.ts b/libs/growth/src/index.ts index 8d159f780..70d278c65 100644 --- a/libs/growth/src/index.ts +++ b/libs/growth/src/index.ts @@ -12,3 +12,35 @@ export * from './lib/scoring.ts'; export * from './lib/stops.ts'; export * from './lib/tokens.ts'; export * from './lib/webhooks.ts'; +export { + parseCollectionBatch, + collectionSource, + ObservationError, + MAX_BODY_BYTES, +} from './lib/observability/contracts.ts'; +export type { + CollectionSource, + CollectionBatchV1, + CollectionEventV1, + CollectionAcknowledgment, +} from './lib/observability/contracts.ts'; +export { acceptObservationBatch } from './lib/observability/ingest.ts'; +export { + consumeSourceBudget, + consumeSubjectBudgets, +} from './lib/observability/admission.ts'; +export { processObservations } from './lib/observability/projection.ts'; +export { processInstallRuntimeActivations } from './lib/observability/install-runtime.ts'; +export { + readTimeline, + readObservationIdentity, + readObservationHealth, +} from './lib/observability/queries.ts'; +export type { TimelineObservation } from './lib/observability/queries.ts'; +export { projectFormObservations } from './lib/observability/form-projection.ts'; +export { replayObservations } from './lib/observability/replay.ts'; +export { + redactObservationEvidence, + initializeObservationRedactions, +} from './lib/observability/redaction.ts'; +export type { ObservationEnrichmentReference } from './lib/observability/enrichment-contract.ts'; diff --git a/libs/growth/src/lib/contacts.spec.ts b/libs/growth/src/lib/contacts.spec.ts index 0a13aab92..0f6fde79f 100644 --- a/libs/growth/src/lib/contacts.spec.ts +++ b/libs/growth/src/lib/contacts.spec.ts @@ -3,8 +3,17 @@ import type { SqlQueryResult, SqlTransaction, } from './database.ts'; +// The observation transaction is exercised against Neon in observability-redaction.integration.spec.ts. +vi.mock('./observability/redaction.ts', () => ({ + redactContactObservationEvidence: vi.fn(async () => undefined), +})); +vi.mock('./observability/store.ts', () => ({ + privacyLock: vi.fn(async () => undefined), +})); import { approveContactFromForm, + approveContactFromInstallRuntimeInTransaction, + normalizeInstallRuntimeEmail, CONTACT_HARD_STOP_REASONS, deleteContact, reauthorizeContact, @@ -124,6 +133,220 @@ function contactRow(overrides: TestRow = {}): TestRow { }; } +describe('install/runtime contact approval', () => { + const input = { + email: ' Person@Example.COM ', + keyring, + now: occurredAt, + installObservationId: '11111111-1111-4111-8111-111111111111', + runtimeObservationId: '22222222-2222-4222-8222-222222222222', + }; + const id = String(contactRow().id); + + it.each([ + '', + 'bad', + 'person@localhost', + 'a..b@example.com', + 'person@-example.com', + 'no-reply@example.com', + 'noreply+tag@example.com', + 'do_not_reply@example.com', + '123+person@users.noreply.github.com', + 'dependabot@example.com', + 'renovate[bot]@example.com', + 'github-actions@example.com', + 'bot+build@example.com', + ])( + 'excludes unusable or automated email %s before database access', + async (email) => { + expect(normalizeInstallRuntimeEmail(email)).toBeNull(); + const harness = executorWith({}); + expect( + await approveContactFromInstallRuntimeInTransaction(harness.executor, { + ...input, + email, + }) + ).toBeNull(); + expect(harness.calls).toEqual([]); + } + ); + + it('normalizes a personal address without inferring company membership', () => { + expect(normalizeInstallRuntimeEmail(' Person+Threadplane@Gmail.COM ')).toBe( + 'person+threadplane@gmail.com' + ); + }); + + it('creates a contact and approves it with only linked-install provenance', async () => { + const harness = executorWith({ + 'lock-email': () => ({ rows: [] }), + 'find-install-runtime-contact': () => ({ rows: [] }), + 'insert-install-runtime-contact': () => ({ rows: [contactRow()] }), + 'find-hard-stops': () => ({ rows: [] }), + 'insert-activity': () => ({ rows: [{ event_key: 'inserted' }] }), + 'set-install-runtime-approval': () => ({ rows: [{ id }] }), + }); + expect( + await approveContactFromInstallRuntimeInTransaction( + harness.executor, + input + ) + ).toBe(id); + expect(harness.transactions.count).toBe(0); + const creation = harness.calls.find( + (call) => call.marker === 'insert-install-runtime-contact' + ); + expect(creation?.parameters).toEqual([ + 'person@example.com', + createEmailLookupHmac(input.email, keyring.active).digest, + 2, + ]); + const activity = harness.calls.find( + (call) => call.marker === 'insert-activity' + ); + expect(activity?.parameters).toEqual([ + `install_runtime.outreach_approved:${id}`, + id, + occurredAt, + 'install_runtime.outreach_approved', + JSON.stringify({ + provenance: 'linked_install_runtime', + install_observation_id: input.installObservationId, + runtime_observation_id: input.runtimeObservationId, + }), + null, + ]); + expect( + harness.calls.find( + (call) => call.marker === 'set-install-runtime-approval' + )?.sql + ).toContain('outreach_approved_at is null'); + expect(harness.calls.some((call) => call.marker.includes('form'))).toBe( + false + ); + }); + + it('approves an existing unapproved contact without overwriting contact facts', async () => { + const harness = executorWith({ + 'lock-email': () => ({ rows: [] }), + 'find-install-runtime-contact': () => ({ rows: [contactRow()] }), + 'find-hard-stops': () => ({ rows: [] }), + 'insert-activity': () => ({ rows: [{ event_key: 'inserted' }] }), + 'set-install-runtime-approval': () => ({ rows: [{ id }] }), + }); + expect( + await approveContactFromInstallRuntimeInTransaction( + harness.executor, + input + ) + ).toBe(id); + expect( + harness.calls.filter((call) => + /insert.*contact|update-contact-facts/.test(call.marker) + ) + ).toEqual([]); + }); + + it('returns an existing approval without another event or timestamp change', async () => { + const harness = executorWith({ + 'lock-email': () => ({ rows: [] }), + 'find-install-runtime-contact': () => ({ + rows: [contactRow({ outreach_approved_at: occurredAt })], + }), + 'find-hard-stops': () => ({ rows: [] }), + }); + expect( + await approveContactFromInstallRuntimeInTransaction( + harness.executor, + input + ) + ).toBe(id); + expect( + harness.calls.some( + (call) => + call.marker === 'insert-activity' || + call.marker === 'set-install-runtime-approval' + ) + ).toBe(false); + }); + + it.each(CONTACT_HARD_STOP_REASONS)( + 'never reauthorizes a contact stopped for %s', + async (reason) => { + const harness = executorWith({ + 'lock-email': () => ({ rows: [] }), + 'find-install-runtime-contact': () => ({ + rows: [ + contactRow({ + outreach_approved_at: new Date('2026-08-01T00:00:00Z'), + }), + ], + }), + 'find-hard-stops': () => ({ + rows: [{ kind: reason, occurred_at: occurredAt }], + }), + }); + expect( + await approveContactFromInstallRuntimeInTransaction( + harness.executor, + input + ) + ).toBeNull(); + expect( + harness.calls.some( + (call) => + call.marker === 'insert-activity' || + call.marker === 'set-install-runtime-approval' + ) + ).toBe(false); + } + ); + + it('keeps a deleted contact ineligible when its email survives only as a HMAC', async () => { + const harness = executorWith({ + 'lock-email': () => ({ rows: [] }), + 'find-install-runtime-contact': () => ({ + rows: [contactRow({ deleted_at: occurredAt })], + }), + 'find-hard-stops': () => ({ rows: [] }), + }); + expect( + await approveContactFromInstallRuntimeInTransaction( + harness.executor, + input + ) + ).toBeNull(); + expect( + harness.calls.find( + (call) => call.marker === 'find-install-runtime-contact' + )?.sql + ).toContain('contact.lookup_alias_added'); + }); + + it('fails closed if the keyring cannot look up an older deleted contact', async () => { + const harness = executorWith({ + 'lock-email': () => ({ rows: [] }), + 'read-key-versions': () => ({ rows: [{ email_hmac_key_version: 3 }] }), + }); + await expect( + approveContactFromInstallRuntimeInTransaction(harness.executor, input) + ).rejects.toThrow(/coverage/); + }); + + it('rejects ambiguous matches instead of approving an arbitrary contact', async () => { + const harness = executorWith({ + 'lock-email': () => ({ rows: [] }), + 'find-install-runtime-contact': () => ({ + rows: [contactRow(), contactRow({ id: 'other' })], + }), + }); + await expect( + approveContactFromInstallRuntimeInTransaction(harness.executor, input) + ).rejects.toThrow(/multiple/); + }); +}); + describe('approveContactFromForm', () => { it('normalizes direct facts, preserves a private lookup, and records exact approval provenance', async () => { const harness = executorWith({ diff --git a/libs/growth/src/lib/contacts.ts b/libs/growth/src/lib/contacts.ts index d117b89a7..5cd0c9118 100644 --- a/libs/growth/src/lib/contacts.ts +++ b/libs/growth/src/lib/contacts.ts @@ -1,8 +1,11 @@ import type { SqlExecutor, SqlTransaction } from './database.ts'; +import { redactContactObservationEvidence } from './observability/redaction.ts'; +import { privacyLock } from './observability/store.ts'; import { compareEmailLookupHmac, createEmailLookupCandidates, normalizeEmail, + normalizeRecipientEmail, type EmailHmacKeyring, } from './crypto.ts'; import type { @@ -143,6 +146,185 @@ export interface FormSubmittedFacts { timeline?: 'this_quarter' | 'next_quarter' | '6_plus_months' | 'exploring'; } +export interface ApproveContactFromInstallRuntimeInput { + email: string; + keyring: EmailHmacKeyring; + now: Date; + installObservationId: string; + runtimeObservationId: string; +} + +/** A usable recipient hint, not verification of ownership or employment. */ +export function normalizeInstallRuntimeEmail(email: string): string | null { + let normalized: string; + try { + normalized = normalizeRecipientEmail(email); + } catch { + return null; + } + const [local, domain] = normalized.split('@'); + if ( + !local || + !domain || + local.length > 64 || + local.startsWith('.') || + local.endsWith('.') || + local.includes('..') || + /[(),:;\\[\]"]/u.test(local) || + !domain + .split('.') + .every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u.test(label)) + ) + return null; + const mailbox = local.split('+')[0].replace(/[._-]/gu, ''); + if ( + [ + 'noreply', + 'donotreply', + 'noreplies', + 'mailerdaemon', + 'bot', + 'buildbot', + 'dependabot', + 'renovate', + 'githubactions', + 'gitlabci', + 'jenkins', + ].includes(mailbox) || + domain.split('.').some((label) => ['noreply', 'no-reply'].includes(label)) + ) + return null; + return normalized; +} + +/** Called only after the server resolves eligible, non-conflicting install/runtime evidence. */ +export async function approveContactFromInstallRuntimeInTransaction( + transaction: SqlTransaction, + input: ApproveContactFromInstallRuntimeInput +): Promise { + const email = normalizeInstallRuntimeEmail(input.email); + if (!email) return null; + const now = validDate('now', input.now); + const observationId = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; + if ( + !observationId.test(input.installObservationId) || + !observationId.test(input.runtimeObservationId) + ) { + throw new Error('Install/runtime approval requires observation UUIDs'); + } + const candidates = createEmailLookupCandidates(email, input.keyring); + const active = candidates[0]; + await privacyLock(transaction); + await transaction.execute( + `/* growth:lock-email */ select pg_advisory_xact_lock(hashtextextended($1, 0))`, + [email] + ); + // Missing rotation keys must not turn a deleted contact into a new eligible identity. + const storedVersions = await transaction.execute<{ + email_hmac_key_version: number; + }>( + `/* growth:read-key-versions */ + select distinct email_hmac_key_version from growth_contacts order by email_hmac_key_version` + ); + if ( + storedVersions.rows.some( + (row) => + !candidates.some( + (candidate) => candidate.keyVersion === row.email_hmac_key_version + ) + ) + ) { + throw new Error( + 'Email HMAC rotation coverage error for install/runtime approval' + ); + } + const found = await transaction.execute( + `/* growth:find-install-runtime-contact */ + select c.id, c.email_lookup_hmac, c.email_hmac_key_version, + c.outreach_approved_at, c.deleted_at, c.updated_at + from growth_contacts c + where c.email_normalized = $2 or exists ( + select 1 from jsonb_to_recordset($1::jsonb) + as candidate(key_version smallint, digest text) + where (candidate.key_version = c.email_hmac_key_version + and candidate.digest = c.email_lookup_hmac) + or exists ( + select 1 from growth_activity alias + where alias.contact_id = c.id and alias.kind = 'contact.lookup_alias_added' + and alias.data->>'key_version' = candidate.key_version::text + and alias.data->>'digest' = candidate.digest + ) + ) + order by c.id limit 2 for update of c`, + [ + JSON.stringify( + candidates.map((candidate) => ({ + key_version: candidate.keyVersion, + digest: candidate.digest, + })) + ), + email, + ] + ); + if (found.rows.length > 1) + throw new Error('Email HMAC lookup matched multiple growth contacts'); + let contact = found.rows[0]; + if (contact) { + const matching = candidates.find( + (candidate) => candidate.keyVersion === contact?.email_hmac_key_version + ); + if ( + !matching || + !compareEmailLookupHmac(matching.digest, contact.email_lookup_hmac) + ) { + throw new Error( + 'Email HMAC secret material is inconsistent for install/runtime approval' + ); + } + } else { + const inserted = await transaction.execute( + `/* growth:insert-install-runtime-contact */ + insert into growth_contacts (email_normalized, email_lookup_hmac, email_hmac_key_version, source) + values ($1, $2, $3, 'install_runtime') + returning id, email_lookup_hmac, email_hmac_key_version, + outreach_approved_at, deleted_at, updated_at`, + [email, active.digest, active.keyVersion] + ); + contact = inserted.rows[0]; + if (!contact) throw new Error('Failed to insert growth contact'); + } + const stops = await findHardStops(transaction, contact.id); + const state = toControlState({ + ...contact, + latest_hard_stop_kind: stops[0]?.kind ?? null, + latest_hard_stop_at: stops[0]?.occurred_at ?? null, + }); + if (state.authorization === 'deleted' || state.authorization === 'stopped') + return null; + if (state.authorization === 'approved') return contact.id; + const approved = await transaction.execute<{ id: string }>( + `/* growth:set-install-runtime-approval */ + update growth_contacts set outreach_approved_at = $2 + where id = $1 and outreach_approved_at is null and deleted_at is null + returning id`, + [contact.id, now] + ); + if (!approved.rows.length) return null; + await insertActivityOnce(transaction, { + eventKey: `install_runtime.outreach_approved:${contact.id}`, + contactId: contact.id, + occurredAt: now, + kind: 'install_runtime.outreach_approved', + data: { + provenance: 'linked_install_runtime', + install_observation_id: input.installObservationId, + runtime_observation_id: input.runtimeObservationId, + }, + }); + return contact.id; +} + export interface ReauthorizeContactInput { contactId: string; eventKey: string; @@ -993,6 +1175,9 @@ export async function deleteContact( }; return executor.transaction(async (transaction) => { + // Form evidence inserts take FK locks on this contact while holding privacy. + // Take privacy first so deletion cannot invert that order. + await privacyLock(transaction, true); const locked = await transaction.execute( `/* growth:lock-contact */ select id, outreach_approved_at, deleted_at, updated_at @@ -1017,6 +1202,7 @@ export async function deleteContact( } await insertActivityOnce(transaction, deletionActivity); + await redactContactObservationEvidence(transaction, contactId, occurredAt); const jobs = await transaction.execute<{ id: string; diff --git a/libs/growth/src/lib/jobs.spec.ts b/libs/growth/src/lib/jobs.spec.ts index 47f4d3061..f450869c7 100644 --- a/libs/growth/src/lib/jobs.spec.ts +++ b/libs/growth/src/lib/jobs.spec.ts @@ -47,6 +47,12 @@ function executorWith( parameters: readonly unknown[] = [] ): Promise> { const marker = /\/\* growth:([a-z0-9-]+) \*\//u.exec(sql)?.[1]; + if ( + sql.includes( + "pg_advisory_xact_lock_shared(hashtextextended('growth-observation-privacy-v1'" + ) + ) + return { rows: [] }; const handler = marker ? handlers[marker] : undefined; if (!marker || !handler) { throw new Error(`Unexpected SQL marker: ${marker ?? 'missing'}`); diff --git a/libs/growth/src/lib/jobs.ts b/libs/growth/src/lib/jobs.ts index c8ed371c8..f94bdc4d3 100644 --- a/libs/growth/src/lib/jobs.ts +++ b/libs/growth/src/lib/jobs.ts @@ -2,6 +2,7 @@ import type { SqlExecutor, SqlTransaction } from './database.ts'; import { CONTACT_HARD_STOP_REASONS } from './contacts.ts'; import { normalizeEmail } from './crypto.ts'; import type { GrowthArtifact, GrowthJob } from './models.ts'; +import { privacyLock } from './observability/store.ts'; const FULFILLMENT_ALLOWED_PRIOR_STOPS = new Set([ 'unsubscribe', @@ -45,6 +46,7 @@ interface ArtifactRow extends Record { } interface LifecycleJobContextRow extends Record { + campaign_enrollment_reason?: string | null; contact_id: string; display_name: string | null; company_name: string | null; @@ -209,6 +211,35 @@ export interface MaterializeCampaignEnrollmentInput { batchSize: number; } +/** Only persisted, still-applicable installation/runtime evidence admits this approval. */ +function installRuntimeApproval(alias: 'approval' | 'authoritative'): string { + return `(${alias}.kind = 'install_runtime.outreach_approved' + and ${alias}.data->>'provenance' = 'linked_install_runtime' + and exists ( + select 1 from growth_install_runtime_links link + join growth_observations i on i.id=link.install_observation_id + join growth_observations r on r.id=link.runtime_observation_id + join growth_observation_identities identity on identity.observation_id=i.id + where link.contact_id=c.id and link.outcome='approved' + and i.id::text=${alias}.data->>'install_observation_id' + and r.id::text=${alias}.data->>'runtime_observation_id' + and i.redacted_at is null and r.redacted_at is null + and i.source='install' and r.source='runtime' and r.kind='runtime.session_started' + and i.properties->>'environment'<>'ci' + and i.installation_token_digest=r.installation_token_digest + and i.properties->>'packageName'=r.properties->>'packageName' + and i.properties->>'packageVersion'=r.properties->>'packageVersion' + and identity.email_normalized=c.email_normalized + and not exists(select 1 from growth_observations removed + where removed.source='install' and removed.installation_token_digest=i.installation_token_digest + and removed.redacted_at is not null) + and not exists(select 1 from growth_observations other + join growth_observation_identities other_identity on other_identity.observation_id=other.id + where other.source='install' and other.installation_token_digest=i.installation_token_digest + and other_identity.email_normalized<>identity.email_normalized) + ))`; +} + export async function materializeCampaignEnrollment( executor: SqlExecutor, input: MaterializeCampaignEnrollmentInput @@ -224,6 +255,7 @@ export async function materializeCampaignEnrollment( } return executor.transaction(async (transaction) => { + await privacyLock(transaction); await transaction.execute( `/* growth:lock-campaign-enrollment */ select pg_advisory_xact_lock( @@ -305,6 +337,7 @@ export async function materializeCampaignEnrollment( approval.kind = 'contact.reauthorized' and approval.data->>'provenance' = 'founder_action' ) + or ${installRuntimeApproval('approval')} ) order by approval.event_key limit 1 @@ -342,6 +375,7 @@ export async function materializeCampaignEnrollment( $2, jsonb_build_object( 'campaign_version', 'v1', + 'enrollment_reason', case when e.approval_kind='install_runtime.outreach_approved' then 'install_runtime' else null end, 'enrollment_start_at', $1::timestamptz, 'approval_event_key', e.approval_event_key, 'approval_kind', e.approval_kind, @@ -514,6 +548,14 @@ export async function leaseDueJobs( ( j.payload->>'step' = '1' and ( + exists ( + select 1 from growth_activity enrollment + where enrollment.event_key='campaign:v1:' || j.contact_id::text || ':enrolled' + and enrollment.contact_id=j.contact_id + and enrollment.data->>'enrollment_reason'='install_runtime' + and enrollment.data->>'approval_kind'='install_runtime.outreach_approved' + ) + or exists ( select 1 from growth_artifacts artifact @@ -569,6 +611,7 @@ export async function leaseDueJobs( } export interface GrowthLifecycleJobContext { + campaignEnrollmentReason?: 'install_runtime' | null; contactId: string; displayName: string | null; companyName: string | null; @@ -593,6 +636,7 @@ export async function readLifecycleJobContext( as email_classification, submission.form_submission, enrollment.occurred_at as enrollment_at, + enrollment.enrollment_reason as campaign_enrollment_reason, artifact.id as artifact_id, artifact.job_id as artifact_job_id, artifact.project_id as artifact_project_id, @@ -625,7 +669,8 @@ export async function readLifecycleJobContext( limit 1 ) submission on true left join lateral ( - select a.occurred_at + select a.occurred_at, case when a.data->>'approval_kind'='install_runtime.outreach_approved' + then a.data->>'enrollment_reason' else null end as enrollment_reason from growth_activity a where a.contact_id = c.id and a.kind = 'campaign.enrolled:v1' @@ -680,6 +725,10 @@ export async function readLifecycleJobContext( : null; return { contactId: row.contact_id, + campaignEnrollmentReason: + row.campaign_enrollment_reason === 'install_runtime' + ? 'install_runtime' + : null, displayName: row.display_name, companyName: row.company_name, companyDomain: row.company_domain, @@ -711,6 +760,7 @@ export async function authorizeLeasedJobForSubmission( } return executor.transaction(async (transaction) => { + await privacyLock(transaction); await transaction.execute( `/* growth:acquire-google-reconcile-advisory-lock */ select pg_advisory_xact_lock(hashtextextended('google-mailbox-reconciliation', 0))` @@ -791,6 +841,7 @@ export async function authorizeLeasedJobForSubmission( authoritative.kind = 'contact.reauthorized' and authoritative.data->>'provenance' = 'founder_action' ) + or ${installRuntimeApproval('authoritative')} ) limit 1 ) approval on true diff --git a/libs/growth/src/lib/observability/admission.spec.ts b/libs/growth/src/lib/observability/admission.spec.ts new file mode 100644 index 000000000..7c0e867e3 --- /dev/null +++ b/libs/growth/src/lib/observability/admission.spec.ts @@ -0,0 +1,27 @@ +import { consumeSourceBudget, consumeSubjectBudgets } from './admission.ts'; +import type { SqlExecutor } from '../database.ts'; + +describe('collection admission', () => { + it('fails closed when storage fails', async () => { + const db = { + transaction: async () => { + throw new Error('secret'); + }, + } as unknown as SqlExecutor; + await expect( + consumeSourceBudget(db, 'install', new Date()) + ).rejects.toThrow('admission_unavailable'); + }); + it('rejects unbounded subject batches before accessing storage', async () => { + const db = { transaction: vi.fn() } as unknown as SqlExecutor; + await expect( + consumeSubjectBudgets( + db, + 'install', + Array(21).fill({ subject: { id: 'x', namespace: 'installation' } }), + new Date() + ) + ).rejects.toThrow(); + expect(db.transaction).not.toHaveBeenCalled(); + }); +}); diff --git a/libs/growth/src/lib/observability/admission.ts b/libs/growth/src/lib/observability/admission.ts new file mode 100644 index 000000000..10c8e1ebe --- /dev/null +++ b/libs/growth/src/lib/observability/admission.ts @@ -0,0 +1,80 @@ +import type { SqlExecutor } from '../database.ts'; +import { + collectionSource, + uuid, + ObservationError, + type CollectionEventV1, + type CollectionSource, +} from './contracts.ts'; + +export interface AdmissionResult { + allowed: boolean; + retryAfterSec: number; +} +async function consume( + db: SqlExecutor, + buckets: { key: string; count: number; limit: number }[], + now: Date +): Promise { + const milliseconds = now.getTime(); + if (!Number.isFinite(milliseconds)) + throw new ObservationError('invalid_payload'); + const window = new Date(Math.floor(milliseconds / 60000) * 60000); + try { + const allowed = await db.transaction(async (tx) => { + let accepted = true; + for (const bucket of buckets.sort((a, b) => a.key.localeCompare(b.key))) { + const result = await tx.execute<{ count: string }>( + `insert into growth_collection_budgets(bucket_key,window_start,count) values($1,$2,$3) + on conflict(bucket_key,window_start) do update set count=growth_collection_budgets.count+excluded.count returning count`, + [bucket.key, window, bucket.count] + ); + if (Number(result.rows[0].count) > bucket.limit) accepted = false; + } + return accepted; + }); + return { + allowed, + retryAfterSec: Math.ceil( + (window.getTime() + 60000 - milliseconds) / 1000 + ), + }; + } catch { + throw new ObservationError('admission_unavailable'); + } +} +export function consumeSourceBudget( + db: SqlExecutor, + source: CollectionSource, + now: Date +): Promise { + collectionSource(source); + return consume(db, [{ key: `source:${source}`, count: 1, limit: 1200 }], now); +} +export async function consumeSubjectBudgets( + db: SqlExecutor, + source: CollectionSource, + events: readonly CollectionEventV1[], + now: Date +): Promise { + collectionSource(source); + if (!events.length || events.length > 20) + throw new ObservationError('invalid_payload'); + const counts = new Map(); + const namespace = { + website: 'website_session', + install: 'installation', + runtime: 'development_browser', + }[source]; + for (const event of events) { + if (event.subject.namespace !== namespace) + throw new ObservationError('invalid_payload'); + const key = `subject:${namespace}:${uuid(event.subject.id)}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return consume( + db, + [...counts].map(([key, count]) => ({ key, count, limit: 120 })), + now + ); +} diff --git a/libs/growth/src/lib/observability/canonical.spec.ts b/libs/growth/src/lib/observability/canonical.spec.ts new file mode 100644 index 000000000..7cd9cf530 --- /dev/null +++ b/libs/growth/src/lib/observability/canonical.spec.ts @@ -0,0 +1,24 @@ +import { canonicalJson, publicDigest, identityDigest } from './canonical.ts'; + +describe('observation digests', () => { + it('is stable across object key order but detects actual content changes', () => { + expect(canonicalJson({ b: 2, a: { d: 4, c: 3 } })).toBe( + canonicalJson({ a: { c: 3, d: 4 }, b: 2 }) + ); + expect(publicDigest({ a: 1 })).not.toBe(publicDigest({ a: 2 })); + }); + it('uses the specified historical key and fails closed without it', () => { + const oldKey = { version: 1, secret: 'a'.repeat(32) }; + const newKey = { version: 2, secret: 'b'.repeat(32) }; + const value = { gitEmail: 'developer@example.invalid' }; + expect( + identityDigest(value, { active: newKey, previous: [oldKey] }, 1) + ).toEqual(identityDigest(value, { active: oldKey })); + expect(identityDigest(value, { active: oldKey })).not.toEqual( + identityDigest(value, { active: newKey }) + ); + expect(() => identityDigest(value, { active: newKey }, 1)).toThrow( + 'identity_key_unavailable' + ); + }); +}); diff --git a/libs/growth/src/lib/observability/canonical.ts b/libs/growth/src/lib/observability/canonical.ts new file mode 100644 index 000000000..b994ec2e9 --- /dev/null +++ b/libs/growth/src/lib/observability/canonical.ts @@ -0,0 +1,42 @@ +import { createHash, createHmac } from 'node:crypto'; +import { + createEmailLookupCandidates, + type EmailHmacKeyring, +} from '../crypto.ts'; +import { ObservationError } from './contracts.ts'; + +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) + return '[' + value.map(canonicalJson).join(',') + ']'; + if (value !== null && typeof value === 'object') { + return ( + '{' + + Object.entries(value) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, item]) => JSON.stringify(key) + ':' + canonicalJson(item)) + .join(',') + + '}' + ); + } + const result = JSON.stringify(value); + if (result === undefined) throw new ObservationError('invalid_payload'); + return result; +} +export function publicDigest(value: unknown): string { + return createHash('sha256').update(canonicalJson(value)).digest('hex'); +} +export function identityDigest( + value: unknown, + keyring: EmailHmacKeyring, + version = keyring.active.version +): string { + createEmailLookupCandidates('key-check@example.invalid', keyring); + const key = [keyring.active, ...(keyring.previous ?? [])].find( + (k) => k.version === version + ); + if (!key) throw new ObservationError('identity_key_unavailable'); + return createHmac('sha256', key.secret) + .update('growth-observation-identity-v1\0') + .update(canonicalJson(value)) + .digest('hex'); +} diff --git a/libs/growth/src/lib/observability/contracts.spec.ts b/libs/growth/src/lib/observability/contracts.spec.ts new file mode 100644 index 000000000..d951d91c1 --- /dev/null +++ b/libs/growth/src/lib/observability/contracts.spec.ts @@ -0,0 +1,235 @@ +import { parseCollectionBatch } from './contracts.ts'; + +export const now = new Date('2026-09-04T12:00:00.000Z'); +export function installFixture() { + return { + schemaVersion: 1, + events: [ + { + eventId: '11111111-1111-4111-8111-111111111111', + kind: 'package.installed', + occurredAt: now.toISOString(), + collectorVersion: '1.0.0', + subject: { + id: '22222222-2222-4222-8222-222222222222', + namespace: 'installation', + scope: 'persistent', + }, + properties: { + packageName: '@threadplane/chat', + packageVersion: '0.0.65', + osFamily: 'linux', + architecture: 'x64', + nodeVersion: '22.0.0', + environment: 'ci', + environmentEvidence: 'github_actions', + ciProvider: 'github_actions', + }, + identity: { + gitEmail: ' Developer@Example.Invalid ', + gitDisplayName: 'Developer', + gitConfigOrigin: 'global', + }, + }, + ], + }; +} + +describe('collection contract', () => { + it.each([ + ['local', 'interactive_package_manager'], + ['ci', 'generic_ci'], + ['unknown', 'unknown'], + ])( + 'retains explicit %s installation evidence', + (environment, environmentEvidence) => { + const batch = installFixture(); + const { ciProvider: _ciProvider, ...rest } = batch.events[0].properties; + const event = { + ...batch.events[0], + properties: { ...rest, environment, environmentEvidence }, + }; + expect( + parseCollectionBatch( + 'install', + { schemaVersion: 1, events: [event] }, + now + ).events[0].properties.environment + ).toBe(environment); + } + ); + it.each([ + 'runtime.session_started', + 'transport.connected', + 'runtime.first_stream_completed', + 'thread.persisted', + 'interrupt.handled', + 'generative_ui.rendered', + ])('accepts the registered runtime kind %s', (kind) => { + const event = { + eventId: installFixture().events[0].eventId, + kind, + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: installFixture().events[0].subject.id, + namespace: 'development_browser', + scope: 'memory', + }, + sessionId: '33333333-3333-4333-8333-333333333333', + properties: { + packageName: '@threadplane/langgraph', + packageVersion: '1', + integration: 'langgraph', + }, + }; + expect( + parseCollectionBatch( + 'runtime', + { schemaVersion: 1, events: [event] }, + now + ).events[0].kind + ).toBe(kind); + }); + it('rejects impossible calendar dates rather than normalizing them to another day', () => { + const batch = installFixture(); + batch.events[0].occurredAt = '2026-09-31T12:00:00Z'; + expect(() => + parseCollectionBatch('install', batch, new Date('2026-10-01T12:00:00Z')) + ).toThrow('invalid_payload'); + }); + it('normalizes install identity without authorizing a contact', () => { + const batch = parseCollectionBatch('install', installFixture(), now); + expect(batch.events[0].identity?.gitEmail).toBe( + 'developer@example.invalid' + ); + expect(batch.events[0]).not.toHaveProperty('trust'); + expect(batch.events[0]).not.toHaveProperty('contactId'); + }); + + it.each(['trust', 'accountId', 'approval', 'receivedAt', 'source'])( + 'rejects a forged %s field without echoing values', + (key) => { + const input = installFixture(); + Object.assign(input.events[0], { [key]: 'DO-NOT-LOG' }); + expect(() => parseCollectionBatch('install', input, now)).toThrow( + 'invalid_payload' + ); + } + ); + + it.each([ + (b: ReturnType) => { + b.events[0].subject.namespace = 'development_browser'; + }, + (b: ReturnType) => { + b.events[0].properties.environment = 'local'; + }, + (b: ReturnType) => { + b.events[0].eventId = 'bad'; + }, + (b: ReturnType) => { + b.events[0].occurredAt = '2026-09-03T11:59:59Z'; + }, + (b: ReturnType) => { + b.events[0].occurredAt = '2026-09-04T12:05:01Z'; + }, + (b: ReturnType) => { + b.events[0].identity.gitDisplayName = 'x\nsecret'; + }, + (b: ReturnType) => { + Object.assign(b.events[0].properties, { cwd: '/private/path' }); + }, + (b: ReturnType) => { + b.events.push(b.events[0]); + }, + ])('rejects invalid combinations and fields', (mutate) => { + const batch = installFixture(); + mutate(batch); + expect(() => parseCollectionBatch('install', batch, now)).toThrow( + 'invalid_payload' + ); + }); + + it('distinguishes unsupported versions and limits the batch', () => { + expect(() => + parseCollectionBatch( + 'install', + { ...installFixture(), schemaVersion: 2 }, + now + ) + ).toThrow('unsupported_version'); + expect(() => + parseCollectionBatch('install', { schemaVersion: 1, events: [] }, now) + ).toThrow('invalid_payload'); + const batch = installFixture(); + batch.events = Array(21).fill(batch.events[0]); + expect(() => parseCollectionBatch('install', batch, now)).toThrow( + 'invalid_payload' + ); + }); + + it('accepts website topics but no identity or full URLs', () => { + const event = { + eventId: installFixture().events[0].eventId, + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: installFixture().events[0].subject.id, + namespace: 'website_session', + scope: 'session', + }, + kind: 'website.content_viewed', + properties: { contentId: 'quickstart', topic: 'getting_started' }, + }; + expect( + parseCollectionBatch( + 'website', + { schemaVersion: 1, events: [event] }, + now + ).events + ).toHaveLength(1); + expect(() => + parseCollectionBatch( + 'website', + { schemaVersion: 1, events: [{ ...event, identity: {} }] }, + now + ) + ).toThrow(); + }); + + it('requires a runtime session and closed milestone properties', () => { + const event = { + eventId: installFixture().events[0].eventId, + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: installFixture().events[0].subject.id, + namespace: 'development_browser', + scope: 'memory', + }, + kind: 'runtime.first_stream_completed', + sessionId: '33333333-3333-4333-8333-333333333333', + properties: { + packageName: '@threadplane/langgraph', + packageVersion: '1', + integration: 'langgraph', + durationBucket: 'lt_1s', + }, + }; + expect( + parseCollectionBatch( + 'runtime', + { schemaVersion: 1, events: [event] }, + now + ).events + ).toHaveLength(1); + expect(() => + parseCollectionBatch( + 'runtime', + { schemaVersion: 1, events: [{ ...event, sessionId: undefined }] }, + now + ) + ).toThrow(); + }); +}); diff --git a/libs/growth/src/lib/observability/contracts.ts b/libs/growth/src/lib/observability/contracts.ts new file mode 100644 index 000000000..cbfda8e2c --- /dev/null +++ b/libs/growth/src/lib/observability/contracts.ts @@ -0,0 +1,357 @@ +import { normalizeEmail } from '../crypto.ts'; + +export const COLLECTION_SOURCES = ['website', 'install', 'runtime'] as const; +export type CollectionSource = (typeof COLLECTION_SOURCES)[number]; +export type ObservationSource = CollectionSource | 'form'; +export type SubjectNamespace = + | 'website_session' + | 'installation' + | 'development_browser'; +export type IdentityScope = 'persistent' | 'session' | 'memory'; +export interface ObservationIdentity { + gitDisplayName?: string; + gitEmail?: string; + gitConfigOrigin?: 'local' | 'global' | 'unknown'; + repositoryProvider?: 'github' | 'gitlab' | 'bitbucket'; + repositoryOwner?: string; +} +export interface CollectionEventV1 { + eventId: string; + kind: string; + occurredAt: string; + collectorVersion: string; + subject: { id: string; namespace: SubjectNamespace; scope: IdentityScope }; + sessionId?: string; + installationToken?: string; + properties: Record; + identity?: ObservationIdentity; +} +export interface CollectionBatchV1 { + schemaVersion: 1; + events: CollectionEventV1[]; +} +export interface CollectionAcknowledgment { + schemaVersion: 1; + events: { + eventId: string; + disposition: 'accepted' | 'duplicate' | 'redacted'; + }[]; +} +export class ObservationError extends Error { + constructor(public readonly code: string) { + super(code); + this.name = 'ObservationError'; + } +} +export const MAX_BATCH_EVENTS = 20; +export const MAX_BODY_BYTES = 65536; +export const MILESTONES = [ + 'transport.connected', + 'runtime.first_stream_completed', + 'thread.persisted', + 'interrupt.handled', + 'generative_ui.rendered', +] as const; +export function invalid(): never { + throw new ObservationError('invalid_payload'); +} +export function uuid(value: unknown): string { + if ( + typeof value !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test( + value + ) + ) + invalid(); + return value.toLowerCase(); +} +function object( + value: unknown, + keys: readonly string[] +): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) invalid(); + if (![Object.prototype, null].includes(Object.getPrototypeOf(value))) + invalid(); + const record = value as Record; + if ( + Reflect.ownKeys(record).some( + (k) => typeof k !== 'string' || !keys.includes(k) + ) + ) + invalid(); + return record; +} +function text(value: unknown, cap: number): string { + if (typeof value !== 'string') invalid(); + for (const character of value) { + const code = character.charCodeAt(0); + if (code < 32 || code === 127) invalid(); + } + const normalized = value.trim().normalize('NFC'); + if (!normalized || normalized.length > cap) invalid(); + return normalized; +} +function oneOf( + value: unknown, + values: readonly T[] +): T { + if (typeof value !== 'string' || !values.includes(value as T)) invalid(); + return value as T; +} +export function collectionSource(value: unknown): CollectionSource { + return oneOf(value, COLLECTION_SOURCES); +} +const packages = [ + '@threadplane/chat', + '@threadplane/langgraph', + '@threadplane/ag-ui', + '@threadplane/render', +]; +const providers = [ + 'generic_ci', + 'github_actions', + 'gitlab_ci', + 'jenkins', + 'travis', + 'circleci', + 'bitbucket', + 'buildkite', +]; +type PropertyValidator = (value: unknown) => string; +type PropertyRule = [PropertyValidator, boolean?]; +const string = + (cap: number): PropertyValidator => + (v) => + text(v, cap); +const campaignToken: PropertyValidator = (value) => { + const token = text(value, 120); + if (!/^[a-z0-9][a-z0-9_-]{0,119}$/u.test(token)) invalid(); + return token; +}; +const enumeration = + (values: readonly string[]): PropertyValidator => + (v) => + oneOf(v, values); +const packageRules: Record = { + packageName: [enumeration(packages)], + packageVersion: [string(64)], +}; +function properties( + value: unknown, + rules: Record +): Record { + const input = object(value, Object.keys(rules)); + const result: Record = {}; + for (const [key, [validate, optional]] of Object.entries(rules)) { + if (!(key in input) && optional) continue; + result[key] = validate(input[key]); + } + return result; +} +function hostname(value: unknown): string { + const host = text(value, 253).toLowerCase(); + if ( + !/^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)*[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u.test( + host + ) + ) + invalid(); + return host; +} +function identity(value: unknown): ObservationIdentity { + const record = object(value, [ + 'gitEmail', + 'gitDisplayName', + 'gitConfigOrigin', + 'repositoryProvider', + 'repositoryOwner', + ]); + const result: ObservationIdentity = {}; + if ('gitEmail' in record) { + try { + result.gitEmail = normalizeEmail(text(record.gitEmail, 320)); + } catch { + invalid(); + } + } + if ('gitDisplayName' in record) + result.gitDisplayName = text(record.gitDisplayName, 160); + if ('gitConfigOrigin' in record) + result.gitConfigOrigin = oneOf(record.gitConfigOrigin, [ + 'local', + 'global', + 'unknown', + ]); + if ('repositoryProvider' in record) + result.repositoryProvider = oneOf(record.repositoryProvider, [ + 'github', + 'gitlab', + 'bitbucket', + ]); + if ('repositoryOwner' in record) { + result.repositoryOwner = text(record.repositoryOwner, 100); + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/u.test(result.repositoryOwner)) + invalid(); + } + if (Boolean(result.repositoryOwner) !== Boolean(result.repositoryProvider)) + invalid(); + return result; +} +function parseEvent( + source: CollectionSource, + input: unknown, + now: Date +): CollectionEventV1 { + const record = object(input, [ + 'eventId', + 'kind', + 'occurredAt', + 'collectorVersion', + 'subject', + 'sessionId', + 'properties', + 'identity', + 'installationToken', + ]); + const subject = object(record.subject, ['id', 'namespace', 'scope']); + const namespace = { + website: 'website_session', + install: 'installation', + runtime: 'development_browser', + } as const; + const occurredAt = text(record.occurredAt, 40); + if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/u.test(occurredAt)) + invalid(); + const time = new Date(occurredAt).getTime(); + if ( + !Number.isFinite(time) || + time < now.getTime() - 86400000 || + time > now.getTime() + 300000 + ) + invalid(); + if (new Date(time).toISOString().slice(0, 19) !== occurredAt.slice(0, 19)) + invalid(); + const event: CollectionEventV1 = { + eventId: uuid(record.eventId), + kind: text(record.kind, 100), + occurredAt: new Date(time).toISOString(), + collectorVersion: text(record.collectorVersion, 64), + subject: { + id: uuid(subject.id), + namespace: oneOf(subject.namespace, [namespace[source]]), + scope: oneOf(subject.scope, ['persistent', 'session', 'memory']), + }, + properties: {}, + }; + if ('sessionId' in record) event.sessionId = uuid(record.sessionId); + if ('installationToken' in record) { + if (source === 'website') invalid(); + event.installationToken = uuid(record.installationToken); + } + if ('identity' in record) { + if (source !== 'install') invalid(); + event.identity = identity(record.identity); + } + if (source === 'install') { + if (event.kind !== 'package.installed' || event.sessionId) invalid(); + event.properties = properties(record.properties, { + ...packageRules, + osFamily: [string(64)], + architecture: [string(64)], + nodeVersion: [string(64)], + environment: [enumeration(['local', 'ci', 'unknown'])], + environmentEvidence: [ + enumeration([...providers, 'interactive_package_manager', 'unknown']), + ], + packageManager: [ + enumeration(['npm', 'pnpm', 'yarn', 'bun', 'unknown']), + true, + ], + packageManagerVersion: [string(64), true], + ciProvider: [enumeration(providers), true], + consumerContext: [enumeration(['checkout', 'unavailable']), true], + }); + const p = event.properties; + if ( + p.ciProvider && + (p.environment !== 'ci' || p.ciProvider !== p.environmentEvidence) + ) + invalid(); + if (p.environment === 'ci' && !providers.includes(p.environmentEvidence)) + invalid(); + if ( + p.environment === 'local' && + p.environmentEvidence !== 'interactive_package_manager' + ) + invalid(); + if (p.environment === 'unknown' && p.environmentEvidence !== 'unknown') + invalid(); + } else if (source === 'runtime') { + if ( + !event.sessionId || + !['runtime.session_started', ...MILESTONES].includes(event.kind) + ) + invalid(); + event.properties = properties(record.properties, { + ...packageRules, + integration: [enumeration(['langgraph', 'ag-ui', 'render'])], + ...(event.kind === 'runtime.first_stream_completed' + ? { + durationBucket: [ + enumeration(['lt_1s', '1s_to_5s', '5s_to_30s', '30s_plus']), + true, + ] as PropertyRule, + } + : {}), + }); + } else { + const kinds: Record> = { + 'website.session_started': { + campaignSource: [campaignToken, true], + campaignMedium: [campaignToken, true], + campaignName: [campaignToken, true], + referrerHost: [hostname, true], + }, + 'website.content_viewed': { + contentId: [string(120)], + topic: [ + enumeration([ + 'getting_started', + 'architecture', + 'comparison', + 'pricing', + 'security', + 'deployment', + 'other', + ]), + ], + }, + 'website.install_command_copied': { + packageName: [enumeration(packages)], + }, + }; + if (!Object.hasOwn(kinds, event.kind)) invalid(); + event.properties = properties(record.properties, kinds[event.kind]); + } + return event; +} +export function parseCollectionBatch( + source: CollectionSource, + value: unknown, + receivedAt: Date +): CollectionBatchV1 { + collectionSource(source); + if (!Number.isFinite(receivedAt.getTime())) invalid(); + const record = object(value, ['schemaVersion', 'events']); + if (record.schemaVersion !== 1) + throw new ObservationError('unsupported_version'); + if ( + !Array.isArray(record.events) || + !record.events.length || + record.events.length > MAX_BATCH_EVENTS + ) + invalid(); + const events = record.events.map((v) => parseEvent(source, v, receivedAt)); + if (new Set(events.map((e) => e.eventId)).size !== events.length) invalid(); + return { schemaVersion: 1, events }; +} diff --git a/libs/growth/src/lib/observability/enrichment-contract.ts b/libs/growth/src/lib/observability/enrichment-contract.ts new file mode 100644 index 000000000..aacedba91 --- /dev/null +++ b/libs/growth/src/lib/observability/enrichment-contract.ts @@ -0,0 +1,16 @@ +/** Internal evidence reference for a later Dawn adapter; never a public collection payload. */ +export type ObservationEnrichmentReference = + | { + subjectId: string; + evaluatedAt: string; + status: 'not_requested' | 'pending' | 'failed'; + } + | { + subjectId: string; + evaluatedAt: string; + status: 'available'; + artifactId: string; + artifactSchemaVersion: number; + applicableObservationIds: readonly string[]; + sourceIds: readonly string[]; + }; diff --git a/libs/growth/src/lib/observability/form-projection.ts b/libs/growth/src/lib/observability/form-projection.ts new file mode 100644 index 000000000..530fb9a0d --- /dev/null +++ b/libs/growth/src/lib/observability/form-projection.ts @@ -0,0 +1,130 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../database.ts'; +import { ObservationError } from './contracts.ts'; +import { publicDigest } from './canonical.ts'; +import { privacyLock } from './store.ts'; + +const UUID = + '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'; +const eligible = `a.kind='contact.form_submission' and a.event_key like 'form:%:accepted' + and a.data->>'provenance'='form_submission' and a.data->>'submission_id' ~ '${UUID}' + and a.event_key='form:' || (a.data->>'submission_id') || ':accepted' + and a.data->>'form_kind' in ('contact','newsletter','whitepaper','pricing') + and (a.data->>'form_kind'<>'whitepaper' or a.data->>'paper' in ('overview','angular','render','chat')) + and c.deleted_at is null + and not exists (select 1 from growth_observations o where o.source='form' and o.event_id::text=a.data->>'submission_id')`; + +export async function readFormProjectionBacklog(db: SqlExecutor) { + const result = await db.execute<{ + count: string; + }>(`select count(*)::text as count from ( + select a.id from growth_activity a join growth_contacts c on c.id=a.contact_id where ${eligible} limit 1001 + ) pending`); + const count = Number(result.rows[0]?.count ?? 0); + return { pending: Math.min(1000, count), capped: count > 1000 }; +} +export async function projectFormObservations( + db: SqlExecutor, + input: { enabled: boolean; limit: number; now?: () => Date } +) { + if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 100) + throw new ObservationError('invalid_payload'); + const result = { projected: 0, disabled: !input.enabled }; + if (!input.enabled) return result; + for (let index = 0; index < input.limit; index++) { + const projected = await db.transaction(async (tx) => { + await privacyLock(tx); + const row = ( + await tx.execute<{ + id: string; + contact_id: string; + occurred_at: Date; + created_at: Date; + data: Record; + }>(` + select a.id,a.contact_id,a.occurred_at,a.created_at,a.data from growth_activity a + join growth_contacts c on c.id=a.contact_id where ${eligible} order by a.id limit 1 for update of a skip locked + `) + ).rows[0]; + if (!row) return false; + const now = input.now?.() ?? new Date(); + if (!Number.isFinite(now.getTime())) + throw new ObservationError('invalid_payload'); + const eventId = row.data.submission_id; + const session = row.data.acquisition_session_id; + const hasSession = + typeof session === 'string' && new RegExp(UUID, 'i').test(session); + const namespace = hasSession ? 'website_session' : 'form_submission'; + const externalId = hasSession ? session.toLowerCase() : eventId; + await tx.execute( + 'insert into growth_observation_subjects(namespace,external_id,first_received_at,last_received_at) values($1,$2,$3,$3) on conflict(namespace,external_id) do nothing', + [namespace, externalId, row.created_at] + ); + const subjectId = ( + await tx.execute<{ id: string }>( + 'select id from growth_observation_subjects where namespace=$1 and external_id=$2 for update', + [namespace, externalId] + ) + ).rows[0].id; + const suppressed = + ( + await tx.execute( + `select 1 from growth_observation_redactions r where + (r.selector_kind='subject' and r.selector_key=$1 and r.key_version=0) or + (r.selector_kind='email' and (r.key_version,r.selector_key) in ( + select email_hmac_key_version,email_lookup_hmac from growth_contacts where id=$2 + union all select (data->>'key_version')::smallint,data->>'digest' from growth_activity where contact_id=$2 and kind='contact.lookup_alias_added' + )) limit 1`, + [subjectId, row.contact_id] + ) + ).rows.length > 0; + const properties = { + formKind: row.data.form_kind, + ...(row.data.form_kind === 'whitepaper' + ? { paper: row.data.paper } + : {}), + }; + const id = randomUUID(); + const digest = publicDigest({ + eventId, + subjectId, + kind: 'form.accepted', + occurredAt: new Date(row.occurred_at).toISOString(), + properties, + }); + const inserted = await tx.execute<{ id: string }>( + `insert into growth_observations(id,source,event_id,subject_id,kind,schema_version,collector_version,identity_scope,occurred_at,received_at,trust,properties,public_digest,redacted_at) + values($1,'form',$2,$3,'form.accepted',1,'form-projector-v1','session',$4,$5,'server_verified',$6::jsonb,$7,$8) + on conflict(source,event_id) do nothing returning id`, + [ + id, + eventId, + subjectId, + row.occurred_at, + row.created_at, + JSON.stringify(properties), + digest, + suppressed ? now : null, + ] + ); + if (!inserted.rows.length) return false; + await tx.execute( + 'update growth_observation_subjects set first_received_at=least(first_received_at,$2),last_received_at=greatest(last_received_at,$2) where id=$1', + [subjectId, row.created_at] + ); + await tx.execute( + 'insert into growth_observation_work(observation_id,available_at,updated_at) values($1,$2,$2)', + [id, now] + ); + if (!suppressed) + await tx.execute( + 'insert into growth_observation_form_links(observation_id,activity_id,contact_id) values($1,$2,$3)', + [id, row.id, row.contact_id] + ); + return true; + }); + if (!projected) break; + result.projected++; + } + return result; +} diff --git a/libs/growth/src/lib/observability/ingest.ts b/libs/growth/src/lib/observability/ingest.ts new file mode 100644 index 000000000..c71960ae4 --- /dev/null +++ b/libs/growth/src/lib/observability/ingest.ts @@ -0,0 +1,176 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../database.ts'; +import { + createEmailLookupCandidates, + createEmailLookupHmac, + type EmailHmacKeyring, +} from '../crypto.ts'; +import { + parseCollectionBatch, + ObservationError, + type CollectionSource, + type CollectionAcknowledgment, +} from './contracts.ts'; +import { identityDigest, publicDigest } from './canonical.ts'; +import { privacyLock, assertIdentityKeyCoverage } from './store.ts'; + +export async function acceptObservationBatch( + db: SqlExecutor, + source: CollectionSource, + input: unknown, + context: { now: Date; keyring?: EmailHmacKeyring } +): Promise { + const batch = parseCollectionBatch(source, input, context.now); + const { now } = context; + const requireKeyring = () => { + if (!context.keyring) + throw new ObservationError('identity_key_unavailable'); + return context.keyring; + }; + return db.transaction(async (tx) => { + await privacyLock(tx); + if (batch.events.some((e) => e.identity)) + await assertIdentityKeyCoverage(tx, requireKeyring()); + const subjectIds = new Map(); + for (const event of [...batch.events].sort((a, b) => + a.subject.id.localeCompare(b.subject.id) + )) { + if (subjectIds.has(event.subject.id)) continue; + await tx.execute( + `insert into growth_observation_subjects(namespace,external_id,first_received_at,last_received_at) values($1,$2,$3,$3) on conflict(namespace,external_id) do nothing`, + [event.subject.namespace, event.subject.id, now] + ); + const row = await tx.execute<{ id: string }>( + 'select id from growth_observation_subjects where namespace=$1 and external_id=$2 for update', + [event.subject.namespace, event.subject.id] + ); + subjectIds.set(event.subject.id, row.rows[0].id); + } + const receipts = new Map< + string, + CollectionAcknowledgment['events'][number] + >(); + for (const event of [...batch.events].sort((a, b) => + a.eventId.localeCompare(b.eventId) + )) { + const { identity, ...publicEvent } = event; + const digest = publicDigest(publicEvent); + const subjectId = subjectIds.get(event.subject.id)!; + const candidates = identity?.gitEmail + ? createEmailLookupCandidates(identity.gitEmail, requireKeyring()) + : []; + const suppressed = await tx.execute( + `select 1 from growth_observation_redactions where + (selector_kind='subject' and selector_key=$1 and key_version=0) or + (selector_kind='email' and (key_version,selector_key) in (select * from unnest($2::smallint[],$3::text[]))) limit 1`, + [ + subjectId, + candidates.map((c) => c.keyVersion), + candidates.map((c) => c.digest), + ] + ); + const redacted = suppressed.rows.length > 0; + const privateHash = + identity && !redacted + ? identityDigest(identity, requireKeyring()) + : null; + const id = randomUUID(); + const inserted = await tx.execute<{ id: string }>( + `insert into growth_observations + (id,source,event_id,subject_id,session_id,kind,schema_version,collector_version,identity_scope,occurred_at,received_at,properties,public_digest,identity_digest,identity_digest_key_version,redacted_at) + values($1,$2,$3,$4,$5,$6,1,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15) + on conflict(source,event_id) do nothing returning id`, + [ + id, + source, + event.eventId, + subjectId, + event.sessionId ?? null, + event.kind, + event.collectorVersion, + event.subject.scope, + event.occurredAt, + now, + JSON.stringify(event.properties), + digest, + privateHash, + privateHash ? requireKeyring().active.version : null, + redacted ? now : null, + ] + ); + if (!inserted.rows.length) { + const old = ( + await tx.execute<{ + public_digest: string; + identity_digest: string | null; + identity_digest_key_version: number | null; + redacted_at: Date | null; + }>( + `select public_digest,identity_digest,identity_digest_key_version,redacted_at from growth_observations where source=$1 and event_id=$2`, + [source, event.eventId] + ) + ).rows[0]; + if (old.public_digest !== digest) + throw new ObservationError('event_conflict'); + if (!old.redacted_at) { + const comparison = identity + ? identityDigest( + identity, + requireKeyring(), + old.identity_digest_key_version ?? + requireKeyring().active.version + ) + : null; + if (comparison !== old.identity_digest) + throw new ObservationError('event_conflict'); + } + receipts.set(event.eventId, { + eventId: event.eventId, + disposition: old.redacted_at ? 'redacted' : 'duplicate', + }); + continue; + } + if (identity && !redacted) { + const email = identity.gitEmail + ? createEmailLookupHmac(identity.gitEmail, requireKeyring().active) + : null; + await tx.execute( + `insert into growth_observation_identities(observation_id,email_normalized,git_display_name,git_config_origin,repository_provider,repository_owner,email_lookup_hmac,email_key_version) + values($1,$2,$3,$4,$5,$6,$7,$8)`, + [ + id, + identity.gitEmail ?? null, + identity.gitDisplayName ?? null, + identity.gitConfigOrigin ?? null, + identity.repositoryProvider ?? null, + identity.repositoryOwner ?? null, + email?.digest ?? null, + email?.keyVersion ?? null, + ] + ); + } + if (event.installationToken) { + await tx.execute( + 'update growth_observations set installation_token_digest=$2 where id=$1', + [id, publicDigest({ installationToken: event.installationToken })] + ); + } + await tx.execute( + `update growth_observation_subjects set first_received_at=least(first_received_at,$2),last_received_at=greatest(last_received_at,$2) where id=$1`, + [subjectId, now] + ); + await tx.execute( + `insert into growth_observation_work(observation_id,available_at,updated_at) values($1,$2,$2)`, + [id, now] + ); + receipts.set(event.eventId, { + eventId: event.eventId, + disposition: redacted ? 'redacted' : 'accepted', + }); + } + return { + schemaVersion: 1, + events: batch.events.map((e) => receipts.get(e.eventId)!), + }; + }); +} diff --git a/libs/growth/src/lib/observability/install-runtime-contract.spec.ts b/libs/growth/src/lib/observability/install-runtime-contract.spec.ts new file mode 100644 index 000000000..357fb07b2 --- /dev/null +++ b/libs/growth/src/lib/observability/install-runtime-contract.spec.ts @@ -0,0 +1,60 @@ +import { parseCollectionBatch } from './contracts.ts'; +const now = new Date('2026-09-04T12:00:00.000Z'); +function installFixture() { + return { + schemaVersion: 1, + events: [ + { + eventId: '11111111-1111-4111-8111-111111111111', + kind: 'package.installed', + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: '22222222-2222-4222-8222-222222222222', + namespace: 'installation', + scope: 'persistent', + }, + properties: { + packageName: '@threadplane/langgraph', + packageVersion: '0.0.65', + osFamily: 'linux', + architecture: 'x64', + nodeVersion: '22', + environment: 'unknown', + environmentEvidence: 'unknown', + }, + }, + ], + }; +} + +describe('installation correlation contract', () => { + const token = '12345678-1234-4123-8123-123456789abc'; + it('retains an optional opaque token outside public properties', () => { + const batch = installFixture(); + const event = { ...batch.events[0], installationToken: token }; + expect( + parseCollectionBatch( + 'install', + { schemaVersion: 1, events: [event] }, + now + ).events[0] + ).toMatchObject({ installationToken: token }); + }); + it.each(['email@example.invalid', '', null, '123'])( + 'rejects malformed token %s', + (installationToken) => { + const batch = installFixture(); + expect(() => + parseCollectionBatch( + 'install', + { + schemaVersion: 1, + events: [{ ...batch.events[0], installationToken }], + }, + now + ) + ).toThrow('invalid_payload'); + } + ); +}); diff --git a/libs/growth/src/lib/observability/install-runtime.ts b/libs/growth/src/lib/observability/install-runtime.ts new file mode 100644 index 000000000..543ad2261 --- /dev/null +++ b/libs/growth/src/lib/observability/install-runtime.ts @@ -0,0 +1,110 @@ +import type { SqlExecutor } from '../database.ts'; +import type { EmailHmacKeyring } from '../crypto.ts'; +import { approveContactFromInstallRuntimeInTransaction } from '../contacts.ts'; +import { privacyLock, assertIdentityKeyCoverage } from './store.ts'; + +/** Resolve admitted evidence in the existing lifecycle tick; never invoked by public payloads. */ +export async function processInstallRuntimeActivations( + db: SqlExecutor, + input: { + enabled: boolean; + limit: number; + now: Date; + keyring: EmailHmacKeyring; + } +): Promise<{ + approved: number; + ineligible: number; + conflicted: number; + disabled: boolean; +}> { + const counts = { + approved: 0, + ineligible: 0, + conflicted: 0, + disabled: !input.enabled, + }; + if (!input.enabled) return counts; + if ( + !Number.isInteger(input.limit) || + input.limit < 1 || + input.limit > 100 || + !Number.isFinite(input.now.getTime()) + ) + throw new Error('invalid_activation_input'); + return db.transaction(async (tx) => { + await privacyLock(tx); + await assertIdentityKeyCoverage(tx, input.keyring); + // Reuse database exclusion, not a new queue or in-memory background task. + await tx.execute( + "select pg_advisory_xact_lock(hashtextextended('growth-install-runtime-v1',0))" + ); + const runtimes = await tx.execute<{ + id: string; + installation_token_digest: string; + properties: Record; + }>( + ` + select r.id,r.installation_token_digest,r.properties from growth_observations r + where r.source='runtime' and r.kind='runtime.session_started' and r.redacted_at is null + and r.installation_token_digest is not null + and not exists(select 1 from growth_install_runtime_links l where l.runtime_observation_id=r.id) + and exists(select 1 from growth_observations i where i.source='install' + and i.installation_token_digest=r.installation_token_digest + and i.properties->>'packageName'=r.properties->>'packageName' + and i.properties->>'packageVersion'=r.properties->>'packageVersion') + order by r.received_at,r.id limit $1 for update of r`, + [input.limit] + ); + for (const runtime of runtimes.rows) { + const installs = await tx.execute<{ + id: string; + email_normalized: string | null; + redacted_at: Date | null; + environment: string; + }>( + ` + select i.id,e.email_normalized,i.redacted_at,i.properties->>'environment' as environment + from growth_observations i left join growth_observation_identities e on e.observation_id=i.id + where i.source='install' and i.installation_token_digest=$1 + and i.properties->>'packageName'=$2 and i.properties->>'packageVersion'=$3 + order by i.received_at,i.id`, + [ + runtime.installation_token_digest, + runtime.properties.packageName, + runtime.properties.packageVersion, + ] + ); + const emails = new Set( + installs.rows.map((i) => i.email_normalized).filter(Boolean) + ); + let outcome: 'approved' | 'ineligible' | 'conflicted' = + emails.size > 1 ? 'conflicted' : 'ineligible'; + const installation = installs.rows.find( + (i) => i.email_normalized && !i.redacted_at && i.environment !== 'ci' + ); + let contactId: string | null = null; + if ( + emails.size === 1 && + installation && + !installs.rows.some((i) => i.redacted_at) + ) { + contactId = await approveContactFromInstallRuntimeInTransaction(tx, { + email: installation.email_normalized!, + keyring: input.keyring, + now: input.now, + installObservationId: installation.id, + runtimeObservationId: runtime.id, + }); + if (contactId) outcome = 'approved'; + } + await tx.execute( + `insert into growth_install_runtime_links(runtime_observation_id,install_observation_id,contact_id,outcome,evaluated_at) + values($1,$2,$3,$4,$5)`, + [runtime.id, installation?.id ?? null, contactId, outcome, input.now] + ); + counts[outcome]++; + } + return counts; + }); +} diff --git a/libs/growth/src/lib/observability/projection.ts b/libs/growth/src/lib/observability/projection.ts new file mode 100644 index 000000000..732b1d09a --- /dev/null +++ b/libs/growth/src/lib/observability/projection.ts @@ -0,0 +1,203 @@ +import type { SqlExecutor } from '../database.ts'; +import { MILESTONES, ObservationError, uuid } from './contracts.ts'; +import { privacyLock } from './store.ts'; + +export const PROJECTION_VERSION = 'observation-facts-v1'; +export interface ObservationLease { + observationId: string; + generation: string; + leaseToken: string; + attempts: number; +} +export async function leaseObservationWork( + db: SqlExecutor, + input: { now: Date; limit: number } +): Promise { + return (await claimObservationWork(db, input)).leases; +} +async function claimObservationWork( + db: SqlExecutor, + input: { now: Date; limit: number } +): Promise<{ leases: ObservationLease[]; failed: number }> { + if ( + !Number.isInteger(input.limit) || + input.limit < 1 || + input.limit > 20 || + !Number.isFinite(input.now.getTime()) + ) + throw new ObservationError('invalid_payload'); + const result = await db.execute<{ + observation_id: string; + generation: string; + lease_token: string; + attempts: number; + exhausted_count: number; + }>( + ` + with exhausted as ( + select observation_id from growth_observation_work where attempts>=5 and (status='pending' or (status='leased' and lease_until<=$1)) + order by observation_id for update skip locked limit $2 + ), failed as ( + update growth_observation_work w set status='failed',lease_token=null,lease_until=null,last_error_code='attempts_exhausted',updated_at=$1 + from exhausted e where w.observation_id=e.observation_id returning w.observation_id + ), due as ( + select observation_id from growth_observation_work where attempts<5 and available_at<=$1 + and (status='pending' or (status='leased' and lease_until<=$1)) + order by available_at,observation_id for update skip locked limit ($2-(select count(*) from failed)) + ), claimed as ( + update growth_observation_work w set status='leased',lease_token=gen_random_uuid(),lease_until=$3,attempts=attempts+1,updated_at=$1 + from due where w.observation_id=due.observation_id returning w.observation_id,w.generation,w.lease_token,w.attempts + ) select claimed.*,counts.exhausted_count from (select count(*)::int as exhausted_count from failed) counts left join claimed on true`, + [input.now, input.limit, new Date(input.now.getTime() + 30000)] + ); + return { + failed: result.rows[0]?.exhausted_count ?? 0, + leases: result.rows + .filter((r) => r.observation_id) + .map((r) => ({ + observationId: r.observation_id, + generation: r.generation, + leaseToken: r.lease_token, + attempts: r.attempts, + })), + }; +} +export async function projectObservation( + db: SqlExecutor, + lease: ObservationLease, + context: { now: () => Date } = { now: () => new Date() } +): Promise<'completed' | 'lease_lost'> { + uuid(lease.observationId); + uuid(lease.leaseToken); + try { + return await db.transaction(async (tx) => { + await privacyLock(tx); + const locked = await tx.execute( + `select observation_id from growth_observation_work where observation_id=$1 and generation=$2 and lease_token=$3 and status='leased' and lease_until>$4 for update`, + [lease.observationId, lease.generation, lease.leaseToken, context.now()] + ); + if (!locked.rows.length) return 'lease_lost'; + const observation = ( + await tx.execute<{ + subject_id: string; + source: string; + kind: string; + occurred_at: Date; + received_at: Date; + }>( + 'select subject_id,source,kind,occurred_at,received_at from growth_observations where id=$1', + [lease.observationId] + ) + ).rows[0]; + const activeDay = new Date( + Math.min( + new Date(observation.occurred_at).getTime(), + new Date(observation.received_at).getTime() + ) + ) + .toISOString() + .slice(0, 10); + const milestone = (MILESTONES as readonly string[]).includes( + observation.kind + ) + ? observation.kind + : null; + await tx.execute( + `insert into growth_observation_facts(observation_id,generation,projection_version,projected_at,active_day,milestone_kind,source,subject_id) + values($1,$2,$3,$4,$5,$6,$7,$8) on conflict(observation_id) do update set generation=excluded.generation,projection_version=excluded.projection_version,projected_at=excluded.projected_at,active_day=excluded.active_day,milestone_kind=excluded.milestone_kind`, + [ + lease.observationId, + lease.generation, + PROJECTION_VERSION, + context.now(), + activeDay, + milestone, + observation.source, + observation.subject_id, + ] + ); + const settled = await tx.execute( + `update growth_observation_work set status='completed',lease_token=null,lease_until=null,last_error_code=null,updated_at=$4,projection_version=$5 + where observation_id=$1 and generation=$2 and lease_token=$3 and lease_until>$4 returning observation_id`, + [ + lease.observationId, + lease.generation, + lease.leaseToken, + context.now(), + PROJECTION_VERSION, + ] + ); + if (!settled.rows.length) throw new ObservationError('lease_lost'); + return 'completed'; + }); + } catch (error) { + if (error instanceof ObservationError && error.code === 'lease_lost') + return 'lease_lost'; + throw error; + } +} +async function failObservation( + db: SqlExecutor, + lease: ObservationLease, + now: Date +): Promise<'retry_scheduled' | 'failed' | 'lease_lost'> { + const delay = [60000, 300000, 1800000][Math.min(lease.attempts - 1, 2)]; + const settled = await db.execute( + `update growth_observation_work set status=$4,lease_token=null,lease_until=null,last_error_code='projection_failed',available_at=$5,updated_at=$6 + where observation_id=$1 and generation=$2 and lease_token=$3 and status='leased' and lease_until>$6 returning observation_id`, + [ + lease.observationId, + lease.generation, + lease.leaseToken, + lease.attempts >= 5 ? 'failed' : 'pending', + new Date(now.getTime() + delay), + now, + ] + ); + if (!settled.rows.length) return 'lease_lost'; + return lease.attempts >= 5 ? 'failed' : 'retry_scheduled'; +} +export async function processObservations( + db: SqlExecutor, + input: { enabled: boolean; limit: number; now?: () => Date } +) { + if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 100) + throw new ObservationError('invalid_payload'); + const counts = { + completed: 0, + leaseLost: 0, + retryScheduled: 0, + failed: 0, + disabled: !input.enabled, + }; + if (!input.enabled) return counts; + const now = input.now ?? (() => new Date()); + let remaining = input.limit; + while (remaining > 0) { + const claimed = await claimObservationWork(db, { + now: now(), + limit: Math.min(20, remaining), + }); + const { leases } = claimed; + counts.failed += claimed.failed; + if (!leases.length && !claimed.failed) break; + remaining -= leases.length + claimed.failed; + for (const lease of leases) { + try { + const result = await projectObservation(db, lease, { now }); + if (result === 'completed') counts.completed++; + else counts.leaseLost++; + } catch { + const result = await failObservation(db, lease, now()); + if (result === 'retry_scheduled') counts.retryScheduled++; + else if (result === 'failed') counts.failed++; + else counts.leaseLost++; + } + } + } + await db.execute( + `delete from growth_collection_budgets where (bucket_key,window_start) in (select bucket_key,window_start from growth_collection_budgets where window_start<$1 order by window_start limit 1000)`, + [new Date(now().getTime() - 7200000)] + ); + return counts; +} diff --git a/libs/growth/src/lib/observability/queries.ts b/libs/growth/src/lib/observability/queries.ts new file mode 100644 index 000000000..921d210b4 --- /dev/null +++ b/libs/growth/src/lib/observability/queries.ts @@ -0,0 +1,136 @@ +import type { SqlExecutor } from '../database.ts'; +import { + uuid, + ObservationError, + type ObservationSource, + type IdentityScope, +} from './contracts.ts'; +import { readFormProjectionBacklog } from './form-projection.ts'; + +export type TimelineObservation = { + id: string; + source: ObservationSource; + kind: string; + session_id: string | null; + collector_version: string; + identity_scope: IdentityScope; + occurred_at: Date; + received_at: Date; + trust: 'client_reported' | 'server_verified'; + properties: Record; + identity_redacted: boolean; + processing_status: 'pending' | 'leased' | 'completed' | 'failed'; + last_error_code: string | null; + projection_version: string | null; + active_day: Date | string | null; + milestone_kind: string | null; +}; + +export async function readTimeline( + db: SqlExecutor, + subjectId: string, + input: { limit?: number; cursor?: string } = {} +) { + uuid(subjectId); + const limit = input.limit ?? 100; + if (!Number.isInteger(limit) || limit < 1 || limit > 100) + throw new ObservationError('invalid_payload'); + let cursorTime: Date | null = null, + cursorId: string | null = null; + if (input.cursor) { + try { + if (input.cursor.length > 300) throw new Error(); + const decoded = JSON.parse( + Buffer.from(input.cursor, 'base64url').toString('utf8') + ); + if (!Array.isArray(decoded) || decoded.length !== 2) throw new Error(); + cursorTime = new Date(decoded[0]); + cursorId = uuid(decoded[1]); + if (!Number.isFinite(cursorTime.getTime())) throw new Error(); + } catch { + throw new ObservationError('invalid_cursor'); + } + } + const rows = await db.execute( + `select o.id,o.source,o.kind,o.session_id,o.collector_version,o.identity_scope,o.occurred_at,o.received_at,o.trust,o.properties, + o.redacted_at is not null as identity_redacted,w.status as processing_status,w.last_error_code,f.projection_version,f.active_day,f.milestone_kind + from growth_observations o join growth_observation_work w on w.observation_id=o.id + left join growth_observation_facts f on f.observation_id=o.id + where o.subject_id=$1 and ($2::timestamptz is null or (o.received_at,o.id)>($2,$3::uuid)) + order by o.received_at,o.id limit $4`, + [subjectId, cursorTime, cursorId, limit + 1] + ); + const events = rows.rows.slice(0, limit); + const last = events.at(-1); + const nextCursor = + rows.rows.length > limit && last + ? Buffer.from( + JSON.stringify([new Date(last.received_at).toISOString(), last.id]) + ).toString('base64url') + : null; + return { subjectId, events, nextCursor }; +} +export async function readObservationIdentity( + db: SqlExecutor, + observationId: string +) { + uuid(observationId); + const rows = await db.execute<{ + email_normalized: string | null; + git_display_name: string | null; + git_config_origin: string | null; + repository_provider: string | null; + repository_owner: string | null; + }>( + `select email_normalized,git_display_name,git_config_origin,repository_provider,repository_owner from growth_observation_identities where observation_id=$1`, + [observationId] + ); + if (rows.rows[0]) return rows.rows[0]; + const form = await db.execute<{ + email_normalized: string; + contact_id: string; + provenance: 'form_submission'; + }>( + `select c.email_normalized,c.id as contact_id,'form_submission' as provenance + from growth_observation_form_links l join growth_contacts c on c.id=l.contact_id + where l.observation_id=$1 and c.deleted_at is null`, + [observationId] + ); + return form.rows[0] ?? null; +} +export async function readObservationHealth( + db: SqlExecutor, + input: { from: Date; to: Date } +) { + if ( + !Number.isFinite(input.from.getTime()) || + !Number.isFinite(input.to.getTime()) || + input.to <= input.from || + input.to.getTime() - input.from.getTime() > 31 * 86400000 + ) + throw new ObservationError('invalid_payload'); + const activity = await db.execute( + `select source,kind,collector_version,case when source='install' then properties->>'environment' end as environment, + count(*) as observation_count,count(distinct subject_id) as subject_count,max(received_at) as last_received_at + from growth_observations where received_at>=$1 and received_at<$2 group by source,kind,collector_version,case when source='install' then properties->>'environment' end order by source,kind,collector_version`, + [input.from, input.to] + ); + const work = await db.execute( + 'select * from growth_observation_work_health_v1 order by status,projection_version' + ); + const activation = await db.execute( + `select outcome,count(*) as runtime_count,count(distinct contact_id) as contact_count + from growth_install_runtime_links where evaluated_at >= $1 and evaluated_at < $2 + group by outcome order by outcome`, + [input.from, input.to] + ); + return { + from: input.from.toISOString(), + to: input.to.toISOString(), + activity: activity.rows, + currentQueue: work.rows, + installRuntimeActivation: activation.rows, + formProjection: await readFormProjectionBacklog(db), + ingressFailures: 'service_logs', + }; +} diff --git a/libs/growth/src/lib/observability/redaction.ts b/libs/growth/src/lib/observability/redaction.ts new file mode 100644 index 000000000..7e3e82db7 --- /dev/null +++ b/libs/growth/src/lib/observability/redaction.ts @@ -0,0 +1,214 @@ +import type { SqlExecutor, SqlTransaction } from '../database.ts'; +import { + createEmailLookupCandidates, + normalizeEmail, + type EmailHmacKeyring, + type EmailLookupHmac, +} from '../crypto.ts'; +import { uuid, ObservationError } from './contracts.ts'; +import { identityDigest, publicDigest } from './canonical.ts'; +import { privacyLock } from './store.ts'; + +type Selector = { subjectId: string } | { email: string }; +async function redactLocked( + tx: SqlTransaction, + input: { + subjectId?: string; + email?: string; + lookups: readonly EmailLookupHmac[]; + }, + now: Date +): Promise { + const rows = await tx.execute<{ + observation_id: string; + email_lookup_hmac: string | null; + email_key_version: number | null; + }>( + ` + select o.id as observation_id,coalesce(i.email_lookup_hmac,c.email_lookup_hmac) as email_lookup_hmac, + coalesce(i.email_key_version,c.email_hmac_key_version) as email_key_version from growth_observations o + left join growth_observation_identities i on i.observation_id=o.id + left join growth_observation_form_links l on l.observation_id=o.id + left join growth_contacts c on c.id=l.contact_id where + ($1::uuid is not null and o.subject_id=$1) or + ($2::text is not null and (i.email_normalized=$2 or c.email_normalized=$2)) or + ((i.email_key_version,i.email_lookup_hmac) in (select * from unnest($3::smallint[],$4::text[]))) or + ((c.email_hmac_key_version,c.email_lookup_hmac) in (select * from unnest($3::smallint[],$4::text[]))) order by o.id`, + [ + input.subjectId ?? null, + input.email ?? null, + input.lookups.map((k) => k.keyVersion), + input.lookups.map((k) => k.digest), + ] + ); + if (input.subjectId) + await tx.execute( + `insert into growth_observation_redactions values('subject',$1,0,$2) on conflict do nothing`, + [input.subjectId, now] + ); + const lookups = [ + ...input.lookups, + ...rows.rows.flatMap((r) => + r.email_lookup_hmac && r.email_key_version + ? [{ digest: r.email_lookup_hmac, keyVersion: r.email_key_version }] + : [] + ), + ]; + for (const key of lookups) + await tx.execute( + `insert into growth_observation_redactions values('email',$1,$2,$3) on conflict do nothing`, + [key.digest, key.keyVersion, now] + ); + const ids = rows.rows.map((r) => r.observation_id); + await tx.execute( + 'select observation_id from growth_observation_work where observation_id=any($1::uuid[]) order by observation_id for update', + [ids] + ); + await tx.execute( + 'delete from growth_observation_identities where observation_id=any($1::uuid[])', + [ids] + ); + await tx.execute( + 'delete from growth_observation_form_links where observation_id=any($1::uuid[])', + [ids] + ); + await tx.execute( + 'update growth_observations set identity_digest=null,identity_digest_key_version=null,redacted_at=coalesce(redacted_at,$2) where id=any($1::uuid[])', + [ids, now] + ); + await tx.execute( + 'delete from growth_observation_facts where observation_id=any($1::uuid[])', + [ids] + ); + await tx.execute( + `update growth_observation_work set generation=generation+1,status='pending',attempts=0,lease_token=null,lease_until=null,last_error_code=null,available_at=$2,updated_at=$2 where observation_id=any($1::uuid[])`, + [ids, now] + ); + return ids.length; +} +export async function redactObservationEvidence( + db: SqlExecutor, + selector: Selector, + context: { operationId: string; now: Date; keyring: EmailHmacKeyring } +) { + uuid(context.operationId); + if (!Number.isFinite(context.now.getTime())) + throw new ObservationError('invalid_payload'); + const subjectId = + 'subjectId' in selector ? uuid(selector.subjectId) : undefined; + const email = + 'email' in selector ? normalizeEmail(selector.email) : undefined; + const lookups = email + ? createEmailLookupCandidates(email, context.keyring) + : []; + return db.transaction(async (tx) => { + await privacyLock(tx, true); + // Acquire privacy first everywhere that both locks are needed. + await tx.execute('select pg_advisory_xact_lock(hashtextextended($1,0))', [ + `observation-operation:${context.operationId}`, + ]); + const old = ( + await tx.execute<{ + kind: string; + selection_digest: string; + selected_count: number; + }>( + 'select kind,selection_digest,selected_count from growth_observation_operations where operation_id=$1', + [context.operationId] + ) + ).rows[0]; + if ( + old && + (old.kind !== 'redact' || (email && !/^\d+:/u.test(old.selection_digest))) + ) + throw new ObservationError('operation_conflict'); + const version = + old && email + ? Number(old.selection_digest.split(':')[0]) + : context.keyring.active.version; + const digest = email + ? `${version}:${identityDigest({ email }, context.keyring, version)}` + : publicDigest({ subjectId }); + if (old) { + if (old.kind !== 'redact' || old.selection_digest !== digest) + throw new ObservationError('operation_conflict'); + return { selectedCount: old.selected_count }; + } + const selectedCount = await redactLocked( + tx, + { subjectId, email, lookups }, + context.now + ); + await tx.execute( + `insert into growth_observation_operations values($1,'redact',$2,$3,$4,$2)`, + [context.operationId, context.now, digest, selectedCount] + ); + return { selectedCount }; + }); +} +export async function redactContactObservationEvidence( + tx: SqlTransaction, + contactId: string, + now: Date +): Promise { + await privacyLock(tx, true); + const contact = ( + await tx.execute<{ + email_normalized: string | null; + email_lookup_hmac: string; + email_hmac_key_version: number; + }>( + `/* growth:observation-contact-identity */ select email_normalized,email_lookup_hmac,email_hmac_key_version from growth_contacts where id=$1`, + [contactId] + ) + ).rows[0]; + if (!contact) return; + const aliases = await tx.execute<{ digest: string; version: number }>( + `/* growth:observation-contact-aliases */ select data->>'digest' as digest,(data->>'key_version')::smallint as version from growth_activity where contact_id=$1 and kind='contact.lookup_alias_added'`, + [contactId] + ); + await redactLocked( + tx, + { + email: contact.email_normalized ?? undefined, + lookups: [ + { + digest: contact.email_lookup_hmac, + keyVersion: contact.email_hmac_key_version, + }, + ...aliases.rows.map((k) => ({ + digest: k.digest, + keyVersion: k.version, + })), + ], + }, + now + ); +} +export async function initializeObservationRedactions( + db: SqlExecutor, + input: { limit: number; cursor?: string }, + now = new Date() +) { + if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 100) + throw new ObservationError('invalid_payload'); + const cursor = input.cursor ? uuid(input.cursor) : null; + const rows = await db.execute<{ id: string }>( + 'select id from growth_contacts where deleted_at is not null and ($1::uuid is null or id>$1) order by id limit $2', + [cursor, input.limit + 1] + ); + const selected = rows.rows.slice(0, input.limit); + for (const row of selected) + await db.transaction(async (tx) => { + await privacyLock(tx, true); + await tx.execute( + 'select id from growth_contacts where id=$1 for update', + [row.id] + ); + await redactContactObservationEvidence(tx, row.id, now); + }); + return { + processed: selected.length, + nextCursor: rows.rows.length > input.limit ? selected.at(-1)!.id : null, + }; +} diff --git a/libs/growth/src/lib/observability/replay.ts b/libs/growth/src/lib/observability/replay.ts new file mode 100644 index 000000000..f3279b454 --- /dev/null +++ b/libs/growth/src/lib/observability/replay.ts @@ -0,0 +1,96 @@ +import type { SqlExecutor } from '../database.ts'; +import { + uuid, + collectionSource, + ObservationError, + type CollectionSource, +} from './contracts.ts'; +import { publicDigest } from './canonical.ts'; + +export type ReplaySelection = { operationId: string; maxEvents: number } & ( + | { subjectId: string } + | { source: CollectionSource; from: Date; to: Date } +); +export async function replayObservations( + db: SqlExecutor, + input: ReplaySelection, + now = new Date() +) { + const operationId = uuid(input.operationId); + if ( + !Number.isInteger(input.maxEvents) || + input.maxEvents < 1 || + input.maxEvents > 1000 || + !Number.isFinite(now.getTime()) + ) + throw new ObservationError('invalid_payload'); + let subject: string | null = null, + source: CollectionSource | null = null, + from: Date | null = null, + to: Date | null = null; + if ('subjectId' in input) subject = uuid(input.subjectId); + else { + source = collectionSource(input.source); + from = input.from; + to = input.to; + if ( + !Number.isFinite(from.getTime()) || + !Number.isFinite(to.getTime()) || + to <= from || + to.getTime() - from.getTime() > 86400000 + ) + throw new ObservationError('invalid_payload'); + } + const digest = publicDigest({ + subject, + source, + from: from?.toISOString() ?? null, + to: to?.toISOString() ?? null, + maxEvents: input.maxEvents, + }); + return db.transaction(async (tx) => { + await tx.execute(`select pg_advisory_xact_lock(hashtextextended($1,0))`, [ + `observation-operation:${operationId}`, + ]); + const existing = ( + await tx.execute<{ + kind: string; + selection_digest: string; + selected_count: number; + }>( + 'select kind,selection_digest,selected_count from growth_observation_operations where operation_id=$1', + [operationId] + ) + ).rows[0]; + if (existing) { + if (existing.kind !== 'replay' || existing.selection_digest !== digest) + throw new ObservationError('operation_conflict'); + return { selectedCount: existing.selected_count }; + } + const selected = await tx.execute<{ id: string }>( + `select id from growth_observations where ($1::uuid is not null and subject_id=$1) + or ($1::uuid is null and source=$2 and received_at>=$3 and received_at<$4) order by id limit $5`, + [subject, source, from, to, input.maxEvents + 1] + ); + if (selected.rows.length > input.maxEvents) + throw new ObservationError('selection_overflow'); + const ids = selected.rows.map((r) => r.id); + await tx.execute( + 'select observation_id from growth_observation_work where observation_id=any($1::uuid[]) order by observation_id for update', + [ids] + ); + await tx.execute( + `update growth_observation_work set generation=generation+1,status='pending',attempts=0,lease_token=null,lease_until=null,available_at=$2,updated_at=$2,last_error_code=null where observation_id=any($1::uuid[])`, + [ids, now] + ); + await tx.execute( + 'delete from growth_observation_facts where observation_id=any($1::uuid[])', + [ids] + ); + await tx.execute( + `insert into growth_observation_operations(operation_id,kind,requested_at,selection_digest,selected_count,completed_at) values($1,'replay',$2,$3,$4,$2)`, + [operationId, now, digest, ids.length] + ); + return { selectedCount: ids.length }; + }); +} diff --git a/libs/growth/src/lib/observability/store.ts b/libs/growth/src/lib/observability/store.ts new file mode 100644 index 000000000..e0c47da18 --- /dev/null +++ b/libs/growth/src/lib/observability/store.ts @@ -0,0 +1,40 @@ +import type { SqlTransaction } from '../database.ts'; +import type { EmailHmacKeyring } from '../crypto.ts'; +import { ObservationError } from './contracts.ts'; + +export async function privacyLock( + tx: SqlTransaction, + exclusive = false +): Promise { + await tx.execute( + exclusive + ? `select pg_advisory_xact_lock(hashtextextended('growth-observation-privacy-v1',0))` + : `select pg_advisory_xact_lock_shared(hashtextextended('growth-observation-privacy-v1',0))` + ); +} +export async function assertIdentityKeyCoverage( + tx: SqlTransaction, + keyring: EmailHmacKeyring +): Promise { + const versions = [ + keyring.active.version, + ...(keyring.previous ?? []).map((k) => k.version), + ]; + const missing = await tx.execute( + `select 1 from growth_observation_redactions where selector_kind='email' and not (key_version=any($1::smallint[])) limit 1`, + [versions] + ); + if (missing.rows.length) + throw new ObservationError('identity_key_unavailable'); + const unfenced = await tx.execute(` + with deleted_keys as ( + select email_lookup_hmac as digest,email_hmac_key_version as version from growth_contacts where deleted_at is not null + union all + select a.data->>'digest', (a.data->>'key_version')::smallint from growth_activity a join growth_contacts c on c.id=a.contact_id + where c.deleted_at is not null and a.kind='contact.lookup_alias_added' + ) select 1 from deleted_keys k where not exists ( + select 1 from growth_observation_redactions r where r.selector_kind='email' and r.selector_key=k.digest and r.key_version=k.version + ) limit 1`); + if (unfenced.rows.length) + throw new ObservationError('redaction_initialization_required'); +} diff --git a/libs/growth/src/lib/stops.spec.ts b/libs/growth/src/lib/stops.spec.ts index 404691786..ee2693d88 100644 --- a/libs/growth/src/lib/stops.spec.ts +++ b/libs/growth/src/lib/stops.spec.ts @@ -41,6 +41,12 @@ function executorWith( parameters: readonly unknown[] = [] ): Promise> { const marker = /\/\* growth:([a-z0-9-]+) \*\//u.exec(sql)?.[1]; + if ( + sql.includes( + "pg_advisory_xact_lock_shared(hashtextextended('growth-observation-privacy-v1'" + ) + ) + return { rows: [] }; const handler = marker ? handlers[marker] : undefined; if (marker === 'acquire-google-reconcile-advisory-lock' && !handler) { return { rows: [{}] } as SqlQueryResult; diff --git a/libs/growth/test/contacts.integration.spec.ts b/libs/growth/test/contacts.integration.spec.ts index c5fe3a45b..9a27ac28a 100644 --- a/libs/growth/test/contacts.integration.spec.ts +++ b/libs/growth/test/contacts.integration.spec.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto'; +import { cleanContactObservationFences } from './observability-fixtures.ts'; import { resolve } from 'node:path'; import { @@ -52,6 +53,7 @@ describeDatabase( }); async function removeContact(contactId: string): Promise { + await cleanContactObservationFences(executor, contactId); await executor.execute( `delete from growth_artifacts where contact_id = $1 @@ -666,6 +668,7 @@ describeDatabase( ).toBe(true); } finally { await staleWorker?.close?.(); + await cleanContactObservationFences(executor, contactId); await executor.execute( 'delete from growth_artifacts where project_id = $1', [projectId] diff --git a/libs/growth/test/form-observations.integration.spec.ts b/libs/growth/test/form-observations.integration.spec.ts new file mode 100644 index 000000000..56afaa8ab --- /dev/null +++ b/libs/growth/test/form-observations.integration.spec.ts @@ -0,0 +1,227 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { acceptFormSubmission } from '../src/lib/forms.ts'; +import { deleteContact } from '../src/lib/contacts.ts'; +import { projectFormObservations } from '../src/lib/observability/form-projection.ts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { redactObservationEvidence } from '../src/lib/observability/redaction.ts'; +import { processObservations } from '../src/lib/observability/projection.ts'; +import { readObservationIdentity } from '../src/lib/observability/queries.ts'; +import { replayObservations } from '../src/lib/observability/replay.ts'; +import { + cleanEvidence, + cleanContactObservationFences, + evidenceDatabase, + evidenceKeys, +} from './observability-fixtures.ts'; + +describe('recoverable server form observations', () => { + let db: SqlExecutor; + const subjects: string[] = [], + contacts: string[] = [], + operations: string[] = []; + beforeAll(async () => { + db = await evidenceDatabase(); + }); + afterAll(async () => { + await cleanEvidence(db, subjects, operations); + for (const id of contacts) { + await cleanContactObservationFences(db, id); + await db.execute('delete from growth_activity where contact_id=$1', [id]); + await db.execute('delete from growth_jobs where contact_id=$1', [id]); + await db.execute('delete from growth_contacts where id=$1', [id]); + } + await db.close?.(); + }); + async function form(session: string | undefined = randomUUID()) { + const submissionId = randomUUID(), + email = `${randomUUID()}@example.invalid`, + now = new Date(); + subjects.push(session || submissionId); + const result = await acceptFormSubmission(db, { + submissionId, + email, + form: { kind: 'contact', message: 'DO-NOT-PROJECT' }, + source: 'integration', + sourceForm: 'contact', + noticeText: 'Synthetic notice', + noticeVersion: 'test', + policyVersion: 'test', + acquisitionSessionId: session, + occurredAt: now, + keyring: evidenceKeys, + }); + contacts.push(result.contactId); + return { ...result, session, email, now }; + } + it('recovers after a transaction crash and projects concurrent retries exactly once', async () => { + const accepted = await form(); + const jobs = ( + await db.execute('select id from growth_jobs where contact_id=$1', [ + accepted.contactId, + ]) + ).rows; + const crashing: SqlExecutor = { + execute: db.execute.bind(db), + transaction: (operation) => + db.transaction(async (tx) => { + await operation(tx); + throw new Error('synthetic crash'); + }), + }; + await expect( + projectFormObservations(crashing, { enabled: true, limit: 10 }) + ).rejects.toThrow('synthetic crash'); + expect( + ( + await db.execute( + "select id from growth_observations where source='form' and event_id=$1", + [accepted.submissionId] + ) + ).rows + ).toHaveLength(0); + const results = await Promise.all( + [1, 2].map(() => + projectFormObservations(db, { enabled: true, limit: 10 }) + ) + ); + expect(results.reduce((sum, result) => sum + result.projected, 0)).toBe(1); + const rows = ( + await db.execute( + "select o.id,o.trust,o.properties,s.namespace,s.external_id from growth_observations o join growth_observation_subjects s on s.id=o.subject_id where o.source='form' and o.event_id=$1", + [accepted.submissionId] + ) + ).rows; + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + trust: 'server_verified', + properties: { formKind: 'contact' }, + namespace: 'website_session', + external_id: accepted.session, + }); + expect(JSON.stringify(rows)).not.toMatch(/DO-NOT-PROJECT|notice|email/); + expect( + ( + await db.execute('select id from growth_jobs where contact_id=$1', [ + accepted.contactId, + ]) + ).rows + ).toEqual(jobs); + expect( + (await projectFormObservations(db, { enabled: true, limit: 10 })) + .projected + ).toBe(0); + }); + it('never accepts public form provenance and gives unlinked submissions a separate subject', async () => { + const accepted = await form(''); + await projectFormObservations(db, { enabled: true, limit: 10 }); + const row = ( + await db.execute( + "select s.namespace,s.external_id from growth_observations o join growth_observation_subjects s on s.id=o.subject_id where o.source='form' and o.event_id=$1", + [accepted.submissionId] + ) + ).rows[0]; + expect(row).toEqual({ + namespace: 'form_submission', + external_id: accepted.submissionId, + }); + subjects.push(accepted.submissionId); + await expect( + acceptObservationBatch(db, 'form' as never, {}, { now: new Date() }) + ).rejects.toThrow('invalid_payload'); + }); + it('respects redaction before projection without changing contact approval', async () => { + const accepted = await form(), + operationId = randomUUID(); + operations.push(operationId); + const approval = ( + await db.execute( + 'select outreach_approved_at from growth_contacts where id=$1', + [accepted.contactId] + ) + ).rows; + await redactObservationEvidence( + db, + { email: accepted.email }, + { operationId, now: new Date(), keyring: evidenceKeys } + ); + await projectFormObservations(db, { enabled: true, limit: 10 }); + expect( + ( + await db.execute( + 'select observation_id from growth_observation_form_links where contact_id=$1', + [accepted.contactId] + ) + ).rows + ).toHaveLength(0); + expect( + ( + await db.execute( + 'select outreach_approved_at from growth_contacts where id=$1', + [accepted.contactId] + ) + ).rows + ).toEqual(approval); + }); + it('erases form links on subject redaction and never restores them through replay', async () => { + const accepted = await form(); + await projectFormObservations(db, { enabled: true, limit: 10 }); + const row = ( + await db.execute<{ id: string; subject_id: string }>( + "select id,subject_id from growth_observations where source='form' and event_id=$1", + [accepted.submissionId] + ) + ).rows[0]; + const operationId = randomUUID(), + replayId = randomUUID(); + expect(await readObservationIdentity(db, row.id)).toMatchObject({ + email_normalized: accepted.email, + provenance: 'form_submission', + }); + operations.push(operationId, replayId); + await redactObservationEvidence( + db, + { subjectId: row.subject_id }, + { operationId, now: new Date(), keyring: evidenceKeys } + ); + await replayObservations(db, { + operationId: replayId, + subjectId: row.subject_id, + maxEvents: 10, + }); + await processObservations(db, { enabled: true, limit: 100 }); + await projectFormObservations(db, { enabled: true, limit: 10 }); + expect(await readObservationIdentity(db, row.id)).toBeNull(); + expect( + ( + await db.execute( + 'select observation_id from growth_observation_form_links where observation_id=$1', + [row.id] + ) + ).rows + ).toHaveLength(0); + }); + it('serializes contact deletion with projection without retaining a contact link', async () => { + const accepted = await form(); + await Promise.all([ + projectFormObservations(db, { enabled: true, limit: 10 }), + deleteContact(db, { + contactId: accepted.contactId, + eventKey: `delete:${randomUUID()}`, + occurredAt: new Date(), + actor: 'test', + source: 'integration', + policyVersion: 'test', + }), + ]); + await projectFormObservations(db, { enabled: true, limit: 10 }); + expect( + ( + await db.execute( + 'select observation_id from growth_observation_form_links where contact_id=$1', + [accepted.contactId] + ) + ).rows + ).toHaveLength(0); + }); +}); diff --git a/libs/growth/test/install-runtime.integration.spec.ts b/libs/growth/test/install-runtime.integration.spec.ts new file mode 100644 index 000000000..75041c07b --- /dev/null +++ b/libs/growth/test/install-runtime.integration.spec.ts @@ -0,0 +1,459 @@ +import { randomUUID } from 'node:crypto'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { processInstallRuntimeActivations } from '../src/lib/observability/install-runtime.ts'; +import { + materializeCampaignEnrollment, + leaseDueJobs, + readLifecycleJobContext, + authorizeLeasedJobForSubmission, +} from '../src/lib/jobs.ts'; +import { + evidenceDatabase, + evidenceFixture, + evidenceKeys, +} from './observability-fixtures.ts'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { redactObservationEvidence } from '../src/lib/observability/redaction.ts'; +import { createEmailLookupHmac } from '../src/lib/crypto.ts'; +import { readObservationHealth } from '../src/lib/observability/queries.ts'; +import { privacyLock } from '../src/lib/observability/store.ts'; + +describe('install-runtime founder activation', () => { + let db: SqlExecutor; + const subjects: string[] = [], + emails: string[] = [], + operations: string[] = []; + beforeAll(async () => { + db = await evidenceDatabase(); + }); + afterAll(async () => { + if (!db) return; + const digests = emails.map( + (email) => createEmailLookupHmac(email, evidenceKeys.active).digest + ); + const contacts = ( + await db.execute<{ id: string }>( + 'select id from growth_contacts where email_lookup_hmac=any($1::text[])', + [digests] + ) + ).rows.map((c) => c.id); + await db.execute( + 'delete from growth_observation_subjects where external_id=any($1::uuid[])', + [subjects] + ); + await db.execute( + 'delete from growth_observation_redactions where selector_key=any($1::text[])', + [digests] + ); + await db.execute( + 'delete from growth_observation_operations where operation_id=any($1::uuid[])', + [operations] + ); + await db.execute( + 'delete from growth_activity where contact_id=any($1::uuid[])', + [contacts] + ); + await db.execute( + 'delete from growth_jobs where contact_id=any($1::uuid[])', + [contacts] + ); + await db.execute('delete from growth_contacts where id=any($1::uuid[])', [ + contacts, + ]); + await db.execute( + "delete from growth_activity where event_key='campaign:v1:configuration' and data->>'enrollment_start_at'=$1", + ['2026-01-01T00:00:00+00:00'] + ); + await db.close?.(); + }); + function fixture(now: Date, token = randomUUID()) { + const install = evidenceFixture(now); + install.events[0].installationToken = token; + install.events[0].properties = { + ...install.events[0].properties, + packageName: '@threadplane/langgraph', + packageVersion: '0.0.65', + environment: 'unknown', + environmentEvidence: 'unknown', + }; + const runtime = { + schemaVersion: 1, + events: [ + { + eventId: randomUUID(), + kind: 'runtime.session_started', + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: randomUUID(), + namespace: 'development_browser', + scope: 'persistent', + }, + sessionId: randomUUID(), + installationToken: token, + properties: { + integration: 'langgraph', + packageName: '@threadplane/langgraph', + packageVersion: '0.0.65', + }, + }, + ], + }; + subjects.push(install.events[0].subject.id, runtime.events[0].subject.id); + emails.push(install.events[0].identity!.gitEmail!); + return { install, runtime }; + } + it('resolves a runtime that arrived before install and enrolls once without enrichment', async () => { + const now = new Date(); + const { install, runtime } = fixture(now); + await acceptObservationBatch(db, 'runtime', runtime, { + now, + keyring: evidenceKeys, + }); + expect( + ( + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }) + ).approved + ).toBe(0); + await acceptObservationBatch(db, 'install', install, { + now, + keyring: evidenceKeys, + }); + expect( + ( + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }) + ).approved + ).toBe(1); + expect( + ( + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }) + ).approved + ).toBe(0); + const contact = ( + await db.execute<{ id: string }>( + 'select id from growth_contacts where email_normalized=$1', + [install.events[0].identity!.gitEmail] + ) + ).rows[0]; + const start = new Date('2026-01-01T00:00:00Z'); + await materializeCampaignEnrollment(db, { + enrollmentEnabled: true, + enrollmentStartAt: start, + now, + batchSize: 20, + }); + const jobs = await db.execute<{ id: string }>( + "select id from growth_jobs where contact_id=$1 order by payload->>'step'", + [contact.id] + ); + expect(jobs.rows).toHaveLength(3); + expect( + await readLifecycleJobContext(db, { jobId: jobs.rows[0].id }) + ).toMatchObject({ campaignEnrollmentReason: 'install_runtime' }); + const leased = await leaseDueJobs(db, { + kinds: ['send_step'], + now, + batchSize: 20, + leaseDurationMs: 30000, + campaignEnabled: true, + }); + expect(leased.some((j) => j.id === jobs.rows[0].id)).toBe(true); + const job = leased.find((j) => j.id === jobs.rows[0].id)!; + const operationId = randomUUID(); + operations.push(operationId); + await redactObservationEvidence( + db, + { email: install.events[0].identity!.gitEmail! }, + { operationId, now, keyring: evidenceKeys } + ); + expect( + await authorizeLeasedJobForSubmission(db, { + jobId: job.id, + leaseToken: job.leaseToken!, + now, + campaignEnabled: true, + deliveryEnabled: true, + }) + ).toMatchObject({ authorized: false }); + }); + it.each(['ci', 'noreply', 'conflicting', 'package_mismatch'])( + 'does not approve %s evidence', + async (reason) => { + const now = new Date(); + const { install, runtime } = fixture(now); + if (reason === 'ci') { + install.events[0].properties.environment = 'ci'; + install.events[0].properties.environmentEvidence = 'generic_ci'; + } + if (reason === 'noreply') + install.events[0].identity!.gitEmail = 'noreply@example.invalid'; + if (reason === 'package_mismatch') + runtime.events[0].properties.packageVersion = '0.0.66'; + await acceptObservationBatch(db, 'install', install, { + now, + keyring: evidenceKeys, + }); + if (reason === 'conflicting') { + const other = fixture(now, install.events[0].installationToken).install; + await acceptObservationBatch(db, 'install', other, { + now, + keyring: evidenceKeys, + }); + } + await acceptObservationBatch(db, 'runtime', runtime, { + now, + keyring: evidenceKeys, + }); + expect( + ( + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }) + ).approved + ).toBe(0); + } + ); + it('deduplicates concurrent ticks and additional browsers for the same recipient', async () => { + const now = new Date(); + const { install, runtime } = fixture(now); + await acceptObservationBatch(db, 'install', install, { + now, + keyring: evidenceKeys, + }); + await acceptObservationBatch(db, 'runtime', runtime, { + now, + keyring: evidenceKeys, + }); + const results = await Promise.all( + [1, 2].map(() => + processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }) + ) + ); + expect(results.reduce((sum, r) => sum + r.approved, 0)).toBe(1); + const again = { + ...runtime, + events: [ + { + ...runtime.events[0], + eventId: randomUUID(), + sessionId: randomUUID(), + }, + ], + }; + await acceptObservationBatch(db, 'runtime', again, { + now, + keyring: evidenceKeys, + }); + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }); + const activities = await db.execute( + "select a.id from growth_activity a join growth_contacts c on c.id=a.contact_id where c.email_normalized=$1 and a.kind='install_runtime.outreach_approved'", + [install.events[0].identity!.gitEmail] + ); + expect(activities.rows).toHaveLength(1); + const health = await readObservationHealth(db, { + from: new Date(now.getTime() - 1000), + to: new Date(now.getTime() + 1000), + }); + expect(health).toHaveProperty('installRuntimeActivation'); + expect(JSON.stringify(health)).not.toContain( + install.events[0].installationToken + ); + expect(JSON.stringify(health)).not.toContain( + install.events[0].identity!.gitEmail + ); + }); + it('keeps a wrong-package match pending for a later compatible install', async () => { + const now = new Date(); + const { install, runtime } = fixture(now); + const wrong = { + ...install, + events: [ + { + ...install.events[0], + eventId: randomUUID(), + properties: { + ...install.events[0].properties, + packageVersion: '0.0.64', + }, + }, + ], + }; + await acceptObservationBatch(db, 'install', wrong, { + now, + keyring: evidenceKeys, + }); + await acceptObservationBatch(db, 'runtime', runtime, { + now, + keyring: evidenceKeys, + }); + expect( + ( + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }) + ).approved + ).toBe(0); + await acceptObservationBatch(db, 'install', install, { + now, + keyring: evidenceKeys, + }); + expect( + ( + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }) + ).approved + ).toBe(1); + }); + it('does not revive approval by deleting conflicting source identity', async () => { + const now = new Date(); + const { install, runtime } = fixture(now); + await acceptObservationBatch(db, 'install', install, { + now, + keyring: evidenceKeys, + }); + await acceptObservationBatch(db, 'runtime', runtime, { + now, + keyring: evidenceKeys, + }); + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }); + const other = fixture(now, install.events[0].installationToken).install; + await acceptObservationBatch(db, 'install', other, { + now, + keyring: evidenceKeys, + }); + const operationId = randomUUID(); + operations.push(operationId); + await redactObservationEvidence( + db, + { email: other.events[0].identity!.gitEmail! }, + { operationId, now, keyring: evidenceKeys } + ); + const result = await materializeCampaignEnrollment(db, { + enrollmentEnabled: true, + enrollmentStartAt: new Date('2026-01-01T00:00:00Z'), + now, + batchSize: 100, + }); + const contact = ( + await db.execute<{ id: string }>( + 'select id from growth_contacts where email_normalized=$1', + [install.events[0].identity!.gitEmail] + ) + ).rows[0]; + expect(result.enrolledContactIds).not.toContain(contact.id); + }); + it('waits for a source-redaction transaction before final send authorization', async () => { + const now = new Date(); + const { install, runtime } = fixture(now); + await acceptObservationBatch(db, 'install', install, { + now, + keyring: evidenceKeys, + }); + await acceptObservationBatch(db, 'runtime', runtime, { + now, + keyring: evidenceKeys, + }); + await processInstallRuntimeActivations(db, { + enabled: true, + limit: 20, + now, + keyring: evidenceKeys, + }); + await materializeCampaignEnrollment(db, { + enrollmentEnabled: true, + enrollmentStartAt: new Date('2026-01-01T00:00:00Z'), + now, + batchSize: 100, + }); + const contact = ( + await db.execute<{ id: string }>( + 'select id from growth_contacts where email_normalized=$1', + [install.events[0].identity!.gitEmail] + ) + ).rows[0]; + const job = ( + await leaseDueJobs(db, { + kinds: ['send_step'], + now, + batchSize: 100, + leaseDurationMs: 30000, + campaignEnabled: true, + }) + ).find((j) => j.contactId === contact.id)!; + let release!: () => void, acquired!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const locked = new Promise((resolve) => { + acquired = resolve; + }); + const redaction = db.transaction(async (tx) => { + await privacyLock(tx, true); + acquired(); + await gate; + await tx.execute( + 'update growth_observations set redacted_at=$2 where event_id=$1', + [install.events[0].eventId, now] + ); + }); + await locked; + let settled = false; + const authorization = authorizeLeasedJobForSubmission(db, { + jobId: job.id, + leaseToken: job.leaseToken!, + now, + campaignEnabled: true, + deliveryEnabled: true, + }).finally(() => { + settled = true; + }); + try { + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(settled).toBe(false); + } finally { + release(); + await redaction; + } + expect(await authorization).toMatchObject({ authorized: false }); + }); +}); diff --git a/libs/growth/test/jobs.integration.spec.ts b/libs/growth/test/jobs.integration.spec.ts index 3fbbfd6be..2904d1c4b 100644 --- a/libs/growth/test/jobs.integration.spec.ts +++ b/libs/growth/test/jobs.integration.spec.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto'; +import { cleanContactObservationFences } from './observability-fixtures.ts'; import { resolve } from 'node:path'; import { @@ -47,6 +48,7 @@ describeDatabase( afterEach(async () => { for (const contactId of contactIds) { + await cleanContactObservationFences(executor, contactId); await executor.execute( `delete from growth_artifacts where contact_id = $1 diff --git a/libs/growth/test/migrations.integration.spec.ts b/libs/growth/test/migrations.integration.spec.ts index 3d2635839..b3eeb77a5 100644 --- a/libs/growth/test/migrations.integration.spec.ts +++ b/libs/growth/test/migrations.integration.spec.ts @@ -32,7 +32,7 @@ describeDatabase( await executor?.close?.(); }); - it('applies repeatably and exposes exactly five growth tables and five reporting views', async () => { + it('applies repeatably and exposes the exact growth tables and reporting views', async () => { const directory = resolve(process.cwd(), 'migrations'); await applyMigrations({ directory, executor }); @@ -51,8 +51,18 @@ describeDatabase( expect(tables.rows.map(({ table_name }) => table_name)).toEqual([ 'growth_activity', 'growth_artifacts', + 'growth_collection_budgets', 'growth_contacts', + 'growth_install_runtime_links', 'growth_jobs', + 'growth_observation_facts', + 'growth_observation_form_links', + 'growth_observation_identities', + 'growth_observation_operations', + 'growth_observation_redactions', + 'growth_observation_subjects', + 'growth_observation_work', + 'growth_observations', 'growth_projects', ]); @@ -69,6 +79,9 @@ describeDatabase( 'growth_funnel_daily_v1', 'growth_job_health_v1', 'growth_legacy_progress_v1', + 'growth_observation_source_health_v1', + 'growth_observation_subject_overview_v1', + 'growth_observation_work_health_v1', ]); const ledger = await executor.execute<{ @@ -83,6 +96,10 @@ describeDatabase( { checksum_length: 64, name: '0001_rate_limit_events.sql' }, { checksum_length: 64, name: '0002_growth_control_plane.sql' }, { checksum_length: 64, name: '0003_growth_reporting_views.sql' }, + { checksum_length: 64, name: '0004_growth_observability.sql' }, + { checksum_length: 64, name: '0005_growth_observability_views.sql' }, + { checksum_length: 64, name: '0006_growth_form_observations.sql' }, + { checksum_length: 64, name: '0007_growth_install_runtime.sql' }, ]); }); diff --git a/libs/growth/test/observability-admission.integration.spec.ts b/libs/growth/test/observability-admission.integration.spec.ts new file mode 100644 index 000000000..0ae3db933 --- /dev/null +++ b/libs/growth/test/observability-admission.integration.spec.ts @@ -0,0 +1,80 @@ +import type { SqlExecutor } from '../src/lib/database.ts'; +import { randomUUID } from 'node:crypto'; +import { + consumeSourceBudget, + consumeSubjectBudgets, +} from '../src/lib/observability/admission.ts'; +import type { CollectionEventV1 } from '../src/lib/observability/contracts.ts'; +import { evidenceDatabase } from './observability-fixtures.ts'; + +describe('durable admission limits', () => { + let db: SqlExecutor; + const now = new Date('2001-01-01T00:00:30Z'); + beforeAll(async () => { + db = await evidenceDatabase(); + }); + it('charges every subject on denial and resets at the exact minute boundary', async () => { + const a = randomUUID(), + b = randomUUID(), + start = new Date('2001-01-01T00:00:00Z'); + const event = (id: string) => + ({ + subject: { id, namespace: 'installation', scope: 'persistent' }, + } as CollectionEventV1); + await db.execute( + 'insert into growth_collection_budgets(bucket_key,window_start,count) values($1,$2,119)', + [`subject:installation:${a}`, start] + ); + expect( + ( + await consumeSubjectBudgets( + db, + 'install', + [event(a), event(a), event(b)], + now + ) + ).allowed + ).toBe(false); + expect( + ( + await db.execute( + 'select count from growth_collection_budgets where bucket_key=$1 and window_start=$2', + [`subject:installation:${b}`, start] + ) + ).rows[0].count + ).toBe('1'); + expect( + (await consumeSubjectBudgets(db, 'install', [event(a)], now)).allowed + ).toBe(false); + const next = new Date('2001-01-01T00:01:00Z'); + try { + expect( + await consumeSubjectBudgets(db, 'install', [event(a)], next) + ).toEqual({ allowed: true, retryAfterSec: 60 }); + } finally { + await db.execute( + 'delete from growth_collection_budgets where bucket_key=$1 and window_start=$2', + [`subject:installation:${a}`, next] + ); + } + }); + afterAll(async () => { + await db.execute( + 'delete from growth_collection_budgets where window_start=$1', + [new Date('2001-01-01T00:00:00Z')] + ); + await db.close?.(); + }); + it('admits exactly the remaining quota under concurrent requests', async () => { + await db.execute( + `insert into growth_collection_budgets(bucket_key,window_start,count) values('source:install',$1,1199) on conflict(bucket_key,window_start) do update set count=1199`, + [new Date('2001-01-01T00:00:00Z')] + ); + const results = await Promise.all( + Array.from({ length: 3 }, () => consumeSourceBudget(db, 'install', now)) + ); + expect(results.filter((r) => r.allowed)).toHaveLength(1); + expect(results[0].retryAfterSec).toBe(30); + expect((await consumeSourceBudget(db, 'install', now)).allowed).toBe(false); + }); +}); diff --git a/libs/growth/test/observability-fixtures.ts b/libs/growth/test/observability-fixtures.ts new file mode 100644 index 000000000..4514089b9 --- /dev/null +++ b/libs/growth/test/observability-fixtures.ts @@ -0,0 +1,85 @@ +import { randomUUID } from 'node:crypto'; +import { resolve } from 'node:path'; +import { + createDatabaseExecutor, + type SqlExecutor, +} from '../src/lib/database.ts'; +import type { CollectionBatchV1 } from '../src/lib/observability/contracts.ts'; +// eslint-disable-next-line @nx/enforce-module-boundaries +import { applyMigrations } from '../../../scripts/apply-migrations.mts'; +export const evidenceKeys = { + active: { + version: 777, + secret: 'observation-fixture-secret-at-least-32-bytes', + }, +}; +/** Remove only fences created for this exact synthetic contact before deleting its fixture rows. */ +export async function cleanContactObservationFences( + db: SqlExecutor, + contactId: string +) { + await db.execute( + `delete from growth_observation_redactions r where r.selector_kind='email' and (r.key_version,r.selector_key) in ( + select email_hmac_key_version,email_lookup_hmac from growth_contacts where id=$1 + union all select (data->>'key_version')::smallint,data->>'digest' from growth_activity where contact_id=$1 and kind='contact.lookup_alias_added' + )`, + [contactId] + ); +} +export async function evidenceDatabase(): Promise { + if (!process.env['TEST_DATABASE_URL']) + throw new Error('TEST_DATABASE_URL required'); + const db = createDatabaseExecutor(process.env['TEST_DATABASE_URL']); + await applyMigrations({ directory: resolve('migrations'), executor: db }); + return db; +} +export function evidenceFixture(now = new Date()): CollectionBatchV1 { + return { + schemaVersion: 1, + events: [ + { + eventId: randomUUID(), + kind: 'package.installed', + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: randomUUID(), + namespace: 'installation', + scope: 'persistent', + }, + properties: { + packageName: '@threadplane/chat', + packageVersion: '1', + osFamily: 'linux', + architecture: 'x64', + nodeVersion: '22', + environment: 'ci', + environmentEvidence: 'generic_ci', + }, + identity: { + gitEmail: `${randomUUID()}@example.invalid`, + gitDisplayName: 'Synthetic Developer', + gitConfigOrigin: 'global', + }, + }, + ], + }; +} +export async function cleanEvidence( + db: SqlExecutor, + externalIds: string[], + operationIds: string[] = [] +) { + await db.execute( + `delete from growth_observation_redactions where selector_kind='subject' and selector_key in (select id::text from growth_observation_subjects where external_id=any($1::uuid[]))`, + [externalIds] + ); + await db.execute( + 'delete from growth_observation_subjects where external_id=any($1::uuid[])', + [externalIds] + ); + await db.execute( + 'delete from growth_observation_operations where operation_id=any($1::uuid[])', + [operationIds] + ); +} diff --git a/libs/growth/test/observability-ingest.integration.spec.ts b/libs/growth/test/observability-ingest.integration.spec.ts new file mode 100644 index 000000000..d3102920c --- /dev/null +++ b/libs/growth/test/observability-ingest.integration.spec.ts @@ -0,0 +1,113 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { + cleanEvidence, + evidenceDatabase, + evidenceFixture, + evidenceKeys, +} from './observability-fixtures.ts'; + +describe('durable observation acceptance', () => { + it('accepts multi-subject batches in opposing event order without deadlocks', async () => { + const now = new Date(); + const a = evidenceFixture(now).events[0]; + const b = evidenceFixture(now).events[0]; + subjects.push(a.subject.id, b.subject.id); + await acceptObservationBatch( + db, + 'install', + { schemaVersion: 1, events: [a, b] }, + { now, keyring: evidenceKeys } + ); + const left = structuredClone([a, b]); + const right = structuredClone([b, a]); + left[0].eventId = '00000000-0000-4000-8000-' + randomUUID().slice(-12); + left[1].eventId = 'ffffffff-ffff-4fff-8fff-' + randomUUID().slice(-12); + right[0].eventId = '00000000-0000-4000-8000-' + randomUUID().slice(-12); + right[1].eventId = 'ffffffff-ffff-4fff-8fff-' + randomUUID().slice(-12); + const result = await Promise.allSettled( + [left, right].map((events) => + acceptObservationBatch( + db, + 'install', + { schemaVersion: 1, events }, + { now, keyring: evidenceKeys } + ) + ) + ); + expect(result.map((r) => r.status)).toEqual(['fulfilled', 'fulfilled']); + }); + let db: SqlExecutor; + const subjects: string[] = []; + beforeAll(async () => { + db = await evidenceDatabase(); + }); + afterAll(async () => { + await cleanEvidence(db, subjects); + await db.close?.(); + }); + it('deduplicates concurrent retries without refreshing subject activity', async () => { + const now = new Date(); + const batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + const results = await Promise.all( + Array.from({ length: 3 }, () => + acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }) + ) + ); + expect( + results + .flatMap((r) => r.events) + .filter((e) => e.disposition === 'accepted') + ).toHaveLength(1); + await acceptObservationBatch(db, 'install', batch, { + now: new Date(now.getTime() + 1000), + keyring: evidenceKeys, + }); + const rows = await db.execute<{ total: string; last_received_at: Date }>( + `select count(o.id)::text as total, s.last_received_at from growth_observation_subjects s join growth_observations o on o.subject_id=s.id where s.external_id=$1 group by s.id`, + [subjects.at(-1)] + ); + expect(rows.rows[0].total).toBe('1'); + expect(new Date(rows.rows[0].last_received_at)).toEqual(now); + const identity = await db.execute( + `select i.* from growth_observation_identities i join growth_observations o on o.id=i.observation_id where o.event_id=$1`, + [batch.events[0].eventId] + ); + expect(identity.rows).toHaveLength(1); + }); + it('rolls an entire batch back on a conflicting event', async () => { + const now = new Date(); + const original = evidenceFixture(now); + subjects.push(original.events[0].subject.id); + await acceptObservationBatch(db, 'install', original, { + now, + keyring: evidenceKeys, + }); + const changed = structuredClone(original.events[0]); + changed.properties.packageVersion = '2'; + const fresh = evidenceFixture(now).events[0]; + fresh.eventId = '00000000-0000-4000-8000-' + randomUUID().slice(-12); + subjects.push(fresh.subject.id); + await expect( + acceptObservationBatch( + db, + 'install', + { schemaVersion: 1, events: [fresh, changed] }, + { now, keyring: evidenceKeys } + ) + ).rejects.toThrow('event_conflict'); + expect( + ( + await db.execute( + 'select id from growth_observations where event_id=$1', + [fresh.eventId] + ) + ).rows + ).toEqual([]); + }); +}); diff --git a/libs/growth/test/observability-journey.integration.spec.ts b/libs/growth/test/observability-journey.integration.spec.ts new file mode 100644 index 000000000..ce0026435 --- /dev/null +++ b/libs/growth/test/observability-journey.integration.spec.ts @@ -0,0 +1,245 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { createEmailLookupCandidates } from '../src/lib/crypto.ts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { processObservations } from '../src/lib/observability/projection.ts'; +import { + readTimeline, + readObservationHealth, + readObservationIdentity, +} from '../src/lib/observability/queries.ts'; +import { replayObservations } from '../src/lib/observability/replay.ts'; +import { redactObservationEvidence } from '../src/lib/observability/redaction.ts'; +import { + cleanEvidence, + evidenceDatabase, + evidenceFixture, + evidenceKeys, +} from './observability-fixtures.ts'; + +describe('synthetic evidence journey', () => { + let db: SqlExecutor; + const subjects: string[] = []; + const operations: string[] = []; + let email = ''; + beforeAll(async () => { + db = await evidenceDatabase(); + }); + afterAll(async () => { + await cleanEvidence(db, subjects, operations); + if (email) + for (const key of createEmailLookupCandidates(email, evidenceKeys)) { + await db.execute( + "delete from growth_observation_redactions where selector_kind='email' and selector_key=$1 and key_version=$2", + [key.digest, key.keyVersion] + ); + } + await db.close?.(); + }); + it('accepts three sources, retries, projects, replays and redacts without enrollment', async () => { + const now = new Date(); + const before = ( + await db.execute( + 'select (select count(*) from growth_contacts) as contacts,(select count(*) from growth_jobs) as jobs' + ) + ).rows; + const install = evidenceFixture(now); + email = install.events[0].identity!.gitEmail!; + const website = { + schemaVersion: 1, + events: [ + { + eventId: randomUUID(), + kind: 'website.session_started', + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: randomUUID(), + namespace: 'website_session', + scope: 'session', + }, + properties: {}, + }, + ], + }; + const runtimeSubject = randomUUID(); + const runtime = { + schemaVersion: 1, + events: [ + 'runtime.session_started', + 'transport.connected', + 'runtime.first_stream_completed', + 'runtime.first_stream_completed', + ].map((kind) => ({ + eventId: randomUUID(), + kind, + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: runtimeSubject, + namespace: 'development_browser', + scope: 'memory', + }, + sessionId: randomUUID(), + properties: { + packageName: '@threadplane/langgraph', + packageVersion: '1', + integration: 'langgraph', + }, + })), + }; + subjects.push( + install.events[0].subject.id, + website.events[0].subject.id, + runtimeSubject + ); + for (const [source, batch] of [ + ['website', website], + ['install', install], + ['runtime', runtime], + ] as const) { + const accepted = await acceptObservationBatch(db, source, batch, { + now, + ...(source === 'install' ? { keyring: evidenceKeys } : {}), + }); + expect(accepted.events.every((e) => e.disposition === 'accepted')).toBe( + true + ); + const retry = await acceptObservationBatch(db, source, batch, { + now, + ...(source === 'install' ? { keyring: evidenceKeys } : {}), + }); + expect(retry.events.every((e) => e.disposition === 'duplicate')).toBe( + true + ); + } + const rows = ( + await db.execute<{ id: string; external_id: string }>( + 'select id,external_id from growth_observation_subjects where external_id=any($1::uuid[])', + [subjects] + ) + ).rows; + const runtimeId = rows.find((r) => r.external_id === runtimeSubject)!.id; + const installId = rows.find( + (r) => r.external_id === install.events[0].subject.id + )!.id; + expect( + (await readTimeline(db, runtimeId)).events.every( + (e) => e.processing_status === 'pending' + ) + ).toBe(true); + expect( + (await processObservations(db, { enabled: false, limit: 100 })).disabled + ).toBe(true); + await processObservations(db, { + enabled: true, + limit: 100, + now: () => now, + }); + const timeline = await readTimeline(db, runtimeId, { limit: 2 }); + expect(timeline.events).toHaveLength(2); + expect(timeline.nextCursor).toBeTruthy(); + const second = await readTimeline(db, runtimeId, { + limit: 2, + cursor: timeline.nextCursor!, + }); + expect(second.events).toHaveLength(2); + expect( + new Set([...timeline.events, ...second.events].map((e) => e.id)).size + ).toBe(4); + expect( + [...timeline.events, ...second.events].every( + (e) => e.processing_status === 'completed' + ) + ).toBe(true); + expect(second.nextCursor).toBeNull(); + expect( + ( + await db.execute( + 'select active_days,attained_milestone_count from growth_observation_subject_overview_v1 where subject_id=$1', + [runtimeId] + ) + ).rows[0] + ).toEqual({ active_days: '1', attained_milestone_count: '2' }); + const health = await readObservationHealth(db, { + from: new Date(now.getTime() - 1000), + to: new Date(now.getTime() + 1000), + }); + expect(new Set(health.activity.map((row) => row.source))).toEqual( + new Set(['website', 'install', 'runtime']) + ); + expect(JSON.stringify(health)).not.toContain(email); + const operationId = randomUUID(); + operations.push(operationId); + await expect( + replayObservations( + db, + { subjectId: runtimeId, operationId, maxEvents: 2 }, + now + ) + ).rejects.toThrow('selection_overflow'); + expect( + await replayObservations( + db, + { subjectId: runtimeId, operationId, maxEvents: 4 }, + now + ) + ).toEqual({ selectedCount: 4 }); + await expect( + replayObservations( + db, + { subjectId: installId, operationId, maxEvents: 3 }, + now + ) + ).rejects.toThrow('operation_conflict'); + await processObservations(db, { + enabled: true, + limit: 100, + now: () => now, + }); + const observation = (await readTimeline(db, installId)).events[0]; + expect( + (await readObservationIdentity(db, observation.id))?.email_normalized + ).toBe(email); + const redactionId = randomUUID(); + operations.push(redactionId); + expect( + await redactObservationEvidence( + db, + { email }, + { operationId: redactionId, now, keyring: evidenceKeys } + ) + ).toEqual({ selectedCount: 1 }); + expect( + await redactObservationEvidence( + db, + { email }, + { operationId: redactionId, now, keyring: evidenceKeys } + ) + ).toEqual({ selectedCount: 1 }); + await processObservations(db, { + enabled: true, + limit: 100, + now: () => now, + }); + expect(await readObservationIdentity(db, observation.id)).toBeNull(); + expect(JSON.stringify(await readTimeline(db, installId))).not.toContain( + email + ); + expect( + ( + await db.execute( + 'select (select count(*) from growth_contacts) as contacts,(select count(*) from growth_jobs) as jobs' + ) + ).rows + ).toEqual(before); + expect( + ( + await db.execute( + 'select o.id from growth_observations o join growth_observation_subjects s on s.id=o.subject_id where s.external_id=any($1::uuid[])', + [subjects] + ) + ).rows + ).toHaveLength(6); + }, 60000); +}); diff --git a/libs/growth/test/observability-projection.integration.spec.ts b/libs/growth/test/observability-projection.integration.spec.ts new file mode 100644 index 000000000..cb1343c8e --- /dev/null +++ b/libs/growth/test/observability-projection.integration.spec.ts @@ -0,0 +1,242 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { + leaseObservationWork, + projectObservation, + processObservations, +} from '../src/lib/observability/projection.ts'; +import { replayObservations } from '../src/lib/observability/replay.ts'; +import { + cleanEvidence, + evidenceDatabase, + evidenceFixture, + evidenceKeys, +} from './observability-fixtures.ts'; + +describe('projection ownership', () => { + let db: SqlExecutor; + const subjects: string[] = []; + const operations: string[] = []; + beforeAll(async () => { + db = await evidenceDatabase(); + }); + afterAll(async () => { + await cleanEvidence(db, subjects, operations); + await db.close?.(); + }); + it('processes concurrently without duplicate facts or duplicate settlements', async () => { + const now = new Date(), + a = evidenceFixture(now), + b = evidenceFixture(now); + subjects.push(a.events[0].subject.id, b.events[0].subject.id); + for (const batch of [a, b]) + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const results = await Promise.all( + [1, 2].map(() => + processObservations(db, { enabled: true, limit: 1, now: () => now }) + ) + ); + expect(results.reduce((total, result) => total + result.completed, 0)).toBe( + 2 + ); + const rows = await db.execute( + 'select f.observation_id from growth_observation_facts f join growth_observations o on o.id=f.observation_id where o.event_id=any($1::uuid[])', + [[a.events[0].eventId, b.events[0].eventId]] + ); + expect(rows.rows).toHaveLength(2); + }); + it('reports exhausted crashed leases even when no further work can be claimed', async () => { + const now = new Date(), + batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const id = ( + await db.execute<{ id: string }>( + 'select id from growth_observations where event_id=$1', + [batch.events[0].eventId] + ) + ).rows[0].id; + await db.execute( + "update growth_observation_work set status='leased',attempts=5,lease_token=$2,lease_until=$3 where observation_id=$1", + [id, randomUUID(), now] + ); + expect( + await processObservations(db, { + enabled: true, + limit: 20, + now: () => now, + }) + ).toMatchObject({ failed: 1, retryScheduled: 0 }); + expect( + ( + await db.execute( + 'select status,last_error_code from growth_observation_work where observation_id=$1', + [id] + ) + ).rows[0] + ).toEqual({ status: 'failed', last_error_code: 'attempts_exhausted' }); + }); + it('recovers expired leases and rolls back facts if time expires during settlement', async () => { + const now = new Date(); + const batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const id = ( + await db.execute<{ id: string }>( + 'select id from growth_observations where event_id=$1', + [batch.events[0].eventId] + ) + ).rows[0].id; + const lease = (await leaseObservationWork(db, { now, limit: 20 })).find( + (l) => l.observationId === id + )!; + let calls = 0; + expect( + await projectObservation(db, lease, { + now: () => (++calls < 3 ? now : new Date(now.getTime() + 31000)), + }) + ).toBe('lease_lost'); + expect( + ( + await db.execute( + 'select observation_id from growth_observation_facts where observation_id=$1', + [id] + ) + ).rows + ).toHaveLength(0); + const later = new Date(now.getTime() + 31000); + const replacement = ( + await leaseObservationWork(db, { now: later, limit: 20 }) + ).find((l) => l.observationId === id)!; + expect(replacement.leaseToken).not.toBe(lease.leaseToken); + expect(await projectObservation(db, lease, { now: () => later })).toBe( + 'lease_lost' + ); + expect( + await projectObservation(db, replacement, { now: () => later }) + ).toBe('completed'); + }); + it('isolates a projection failure, schedules a retry, then exhausts attempts', async () => { + const now = new Date(); + const bad = evidenceFixture(now), + good = evidenceFixture(now); + subjects.push(bad.events[0].subject.id, good.events[0].subject.id); + for (const batch of [bad, good]) + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const id = ( + await db.execute<{ id: string }>( + 'select id from growth_observations where event_id=$1', + [bad.events[0].eventId] + ) + ).rows[0].id; + const failing: SqlExecutor = { + execute: db.execute.bind(db), + transaction: (operation) => + db.transaction((tx) => + operation({ + execute: (sql, params) => { + if ( + sql.includes('insert into growth_observation_facts') && + params?.[0] === id + ) + throw new Error('synthetic storage failure'); + return tx.execute(sql, params); + }, + }) + ), + }; + const result = await processObservations(failing, { + enabled: true, + limit: 20, + now: () => now, + }); + expect(result).toMatchObject({ retryScheduled: 1, failed: 0 }); + const retry = ( + await db.execute<{ status: string; available_at: Date }>( + 'select status,available_at from growth_observation_work where observation_id=$1', + [id] + ) + ).rows[0]; + expect(retry.status).toBe('pending'); + expect(new Date(retry.available_at).getTime()).toBe(now.getTime() + 60000); + await db.execute( + 'update growth_observation_work set attempts=4 where observation_id=$1', + [id] + ); + expect( + await processObservations(failing, { + enabled: true, + limit: 20, + now: () => new Date(now.getTime() + 60000), + }) + ).toMatchObject({ failed: 1, retryScheduled: 0 }); + expect( + ( + await db.execute( + 'select status from growth_observation_work where observation_id=$1', + [id] + ) + ).rows[0].status + ).toBe('failed'); + }); + it('fences an old worker after replay and rebuilds one fact', async () => { + const now = new Date(); + const batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const observation = ( + await db.execute<{ id: string; subject_id: string }>( + 'select id, subject_id from growth_observations where event_id=$1', + [batch.events[0].eventId] + ) + ).rows[0]; + const leases = await leaseObservationWork(db, { now, limit: 20 }); + const lease = leases.find((l) => l.observationId === observation.id)!; + expect(lease).toBeDefined(); + const operationId = randomUUID(); + operations.push(operationId); + await replayObservations( + db, + { operationId, subjectId: observation.subject_id, maxEvents: 10 }, + now + ); + expect(await projectObservation(db, lease, { now: () => now })).toBe( + 'lease_lost' + ); + const fresh = (await leaseObservationWork(db, { now, limit: 20 })).find( + (l) => l.observationId === observation.id + )!; + expect(await projectObservation(db, fresh, { now: () => now })).toBe( + 'completed' + ); + await replayObservations( + db, + { operationId, subjectId: observation.subject_id, maxEvents: 10 }, + now + ); + expect( + ( + await db.execute( + 'select * from growth_observation_facts where observation_id=$1', + [observation.id] + ) + ).rows + ).toHaveLength(1); + }); +}); diff --git a/libs/growth/test/observability-redaction.integration.spec.ts b/libs/growth/test/observability-redaction.integration.spec.ts new file mode 100644 index 000000000..17e18908b --- /dev/null +++ b/libs/growth/test/observability-redaction.integration.spec.ts @@ -0,0 +1,356 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { createEmailLookupCandidates } from '../src/lib/crypto.ts'; +import { deleteContact } from '../src/lib/contacts.ts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { replayObservations } from '../src/lib/observability/replay.ts'; +import { + leaseObservationWork, + projectObservation, +} from '../src/lib/observability/projection.ts'; +import { + redactObservationEvidence, + redactContactObservationEvidence, + initializeObservationRedactions, +} from '../src/lib/observability/redaction.ts'; +import { + readTimeline, + readObservationIdentity, +} from '../src/lib/observability/queries.ts'; +import { + cleanEvidence, + evidenceDatabase, + evidenceFixture, + evidenceKeys, +} from './observability-fixtures.ts'; + +describe('private evidence redaction', () => { + let db: SqlExecutor; + const subjects: string[] = []; + const operations: string[] = []; + const emails: string[] = []; + const contacts: string[] = []; + beforeAll(async () => { + db = await evidenceDatabase(); + }); + it('redacts every identity on one subject and fences their future installs', async () => { + const now = new Date(), + first = evidenceFixture(now), + second = evidenceFixture(now); + second.events[0].subject = first.events[0].subject; + subjects.push(first.events[0].subject.id); + for (const batch of [first, second]) { + emails.push(batch.events[0].identity!.gitEmail!); + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + } + const id = ( + await db.execute<{ id: string }>( + 'select id from growth_observation_subjects where external_id=$1', + [first.events[0].subject.id] + ) + ).rows[0].id; + const operationId = randomUUID(); + operations.push(operationId); + expect( + await redactObservationEvidence( + db, + { subjectId: id }, + { operationId, now, keyring: evidenceKeys } + ) + ).toEqual({ selectedCount: 2 }); + expect( + ( + await db.execute( + 'select i.observation_id from growth_observation_identities i join growth_observations o on o.id=i.observation_id where o.subject_id=$1', + [id] + ) + ).rows + ).toHaveLength(0); + for (const batch of [first, second]) { + batch.events[0].eventId = randomUUID(); + batch.events[0].subject = { + ...batch.events[0].subject, + id: randomUUID(), + }; + subjects.push(batch.events[0].subject.id); + expect( + ( + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }) + ).events[0].disposition + ).toBe('redacted'); + } + }); + it('initializes historical deleted-contact fences and aliases repeatably', async () => { + const now = new Date(), + batch = evidenceFixture(now), + alias = evidenceFixture(now); + subjects.push(batch.events[0].subject.id, alias.events[0].subject.id); + const email = batch.events[0].identity!.gitEmail!, + aliasEmail = alias.events[0].identity!.gitEmail!; + emails.push(email, aliasEmail); + const key = createEmailLookupCandidates(email, evidenceKeys)[0], + aliasKey = createEmailLookupCandidates(aliasEmail, evidenceKeys)[0]; + const id = randomUUID(); + contacts.push(id); + await db.execute( + "insert into growth_contacts(id,email_normalized,email_lookup_hmac,email_hmac_key_version,source,deleted_at) values($1,null,$2,$3,'integration',$4)", + [id, key.digest, key.keyVersion, now] + ); + await db.execute( + "insert into growth_activity(contact_id,kind,event_key,occurred_at,data) values($1,'contact.lookup_alias_added',$2,$3,$4::jsonb)", + [ + id, + `fixture:${randomUUID()}`, + now, + JSON.stringify({ + digest: aliasKey.digest, + key_version: aliasKey.keyVersion, + }), + ] + ); + await expect( + acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }) + ).rejects.toThrow('redaction_initialization_required'); + for (let pass = 0; pass < 2; pass++) { + let cursor: string | undefined; + do { + const result = await initializeObservationRedactions( + db, + { limit: 100, cursor }, + now + ); + cursor = result.nextCursor ?? undefined; + } while (cursor); + } + for (const fixture of [batch, alias]) + expect( + ( + await acceptObservationBatch(db, 'install', fixture, { + now, + keyring: evidenceKeys, + }) + ).events[0].disposition + ).toBe('redacted'); + }); + it('rejects operation IDs belonging to replay before interpreting private digests', async () => { + const now = new Date(), + operationId = randomUUID(); + operations.push(operationId); + await replayObservations( + db, + { operationId, subjectId: randomUUID(), maxEvents: 1 }, + now + ); + await expect( + redactObservationEvidence( + db, + { email: 'synthetic@example.invalid' }, + { operationId, now, keyring: evidenceKeys } + ) + ).rejects.toThrow('operation_conflict'); + }); + it('fences leased work and retained keys while leaving another identity untouched', async () => { + const now = new Date(), + batch = evidenceFixture(now), + other = evidenceFixture(now); + subjects.push(batch.events[0].subject.id, other.events[0].subject.id); + const email = batch.events[0].identity!.gitEmail!; + emails.push(email); + for (const fixture of [batch, other]) + await acceptObservationBatch(db, 'install', fixture, { + now, + keyring: evidenceKeys, + }); + const rows = ( + await db.execute<{ id: string; event_id: string }>( + 'select id,event_id from growth_observations where event_id=any($1::uuid[])', + [[batch.events[0].eventId, other.events[0].eventId]] + ) + ).rows; + const id = rows.find((r) => r.event_id === batch.events[0].eventId)!.id; + const otherId = rows.find( + (r) => r.event_id === other.events[0].eventId + )!.id; + const lease = (await leaseObservationWork(db, { now, limit: 20 })).find( + (l) => l.observationId === id + )!; + const operationId = randomUUID(); + operations.push(operationId); + await redactObservationEvidence( + db, + { email }, + { operationId, now, keyring: evidenceKeys } + ); + expect(await projectObservation(db, lease, { now: () => now })).toBe( + 'lease_lost' + ); + expect(await readObservationIdentity(db, id)).toBeNull(); + expect((await readObservationIdentity(db, otherId))?.email_normalized).toBe( + other.events[0].identity!.gitEmail + ); + const rotated = { + active: { + version: 778, + secret: 'rotated-fixture-secret-at-least-32-bytes', + }, + previous: [evidenceKeys.active], + }; + batch.events[0].eventId = randomUUID(); + expect( + ( + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: rotated, + }) + ).events[0].disposition + ).toBe('redacted'); + await expect( + acceptObservationBatch(db, 'install', batch, { + now, + keyring: { active: rotated.active }, + }) + ).rejects.toThrow('identity_key_unavailable'); + }); + afterAll(async () => { + await cleanEvidence(db, subjects, operations); + for (const email of emails) + for (const key of createEmailLookupCandidates(email, evidenceKeys)) + await db.execute( + `delete from growth_observation_redactions where selector_kind='email' and selector_key=$1 and key_version=$2`, + [key.digest, key.keyVersion] + ); + await db.execute( + 'delete from growth_activity where contact_id=any($1::uuid[])', + [contacts] + ); + await db.execute('delete from growth_contacts where id=any($1::uuid[])', [ + contacts, + ]); + await db.close?.(); + }); + it('redacts an install-only identity and never restores it on new events or retries', async () => { + const now = new Date(); + const batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + const email = batch.events[0].identity!.gitEmail!; + emails.push(email); + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const row = ( + await db.execute<{ id: string; subject_id: string }>( + 'select id,subject_id from growth_observations where event_id=$1', + [batch.events[0].eventId] + ) + ).rows[0]; + expect( + JSON.stringify(await readTimeline(db, row.subject_id)) + ).not.toContain(email); + expect((await readObservationIdentity(db, row.id))?.email_normalized).toBe( + email + ); + const operationId = randomUUID(); + operations.push(operationId); + await redactObservationEvidence( + db, + { email }, + { operationId, now, keyring: evidenceKeys } + ); + expect(await readObservationIdentity(db, row.id)).toBeNull(); + expect( + ( + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }) + ).events[0].disposition + ).toBe('redacted'); + batch.events[0].eventId = randomUUID(); + expect( + ( + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }) + ).events[0].disposition + ).toBe('redacted'); + const count = await db.execute( + 'select i.observation_id from growth_observation_identities i join growth_observations o on o.id=i.observation_id where o.subject_id=$1', + [row.subject_id] + ); + expect(count.rows).toEqual([]); + }); + it('fences a contact before its first observation exists', async () => { + const now = new Date(); + const batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + const email = batch.events[0].identity!.gitEmail!; + emails.push(email); + const key = createEmailLookupCandidates(email, evidenceKeys)[0]; + const id = randomUUID(); + contacts.push(id); + await db.execute( + `insert into growth_contacts(id,email_normalized,email_lookup_hmac,email_hmac_key_version,source) values($1,$2,$3,$4,'integration')`, + [id, email, key.digest, key.keyVersion] + ); + await db.transaction(async (tx) => { + await tx.execute( + 'select id from growth_contacts where id=$1 for update', + [id] + ); + await redactContactObservationEvidence(tx, id, now); + await tx.execute( + 'update growth_contacts set email_normalized=null,deleted_at=$2 where id=$1', + [id, now] + ); + }); + expect( + ( + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }) + ).events[0].disposition + ).toBe('redacted'); + }); + it('extends the canonical contact deletion transaction', async () => { + const now = new Date(); + const batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + const email = batch.events[0].identity!.gitEmail!; + emails.push(email); + const key = createEmailLookupCandidates(email, evidenceKeys)[0]; + const id = randomUUID(); + contacts.push(id); + await db.execute( + `insert into growth_contacts(id,email_normalized,email_lookup_hmac,email_hmac_key_version,source) values($1,$2,$3,$4,'integration')`, + [id, email, key.digest, key.keyVersion] + ); + await deleteContact(db, { + contactId: id, + eventKey: `integration:delete:${randomUUID()}`, + occurredAt: now, + actor: 'test', + source: 'integration', + policyVersion: 'growth-v1', + }); + expect( + ( + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }) + ).events[0].disposition + ).toBe('redacted'); + }); +}); diff --git a/libs/growth/test/observability-schema.integration.spec.ts b/libs/growth/test/observability-schema.integration.spec.ts new file mode 100644 index 000000000..9c6e304ae --- /dev/null +++ b/libs/growth/test/observability-schema.integration.spec.ts @@ -0,0 +1,94 @@ +import { resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { + createDatabaseExecutor, + type SqlExecutor, +} from '../src/lib/database.ts'; +// eslint-disable-next-line @nx/enforce-module-boundaries +import { applyMigrations } from '../../../scripts/apply-migrations.mts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { + evidenceFixture, + evidenceKeys, + cleanEvidence, +} from './observability-fixtures.ts'; + +describe('observation schema', () => { + let db: SqlExecutor; + beforeAll(async () => { + db = createDatabaseExecutor(process.env['TEST_DATABASE_URL']); + await applyMigrations({ directory: resolve('migrations'), executor: db }); + }); + afterAll(async () => db.close?.()); + it('installs an isolated queue and redacted reporting', async () => { + const result = await db.execute<{ name: string }>( + `select table_name as name from information_schema.tables where table_schema='public' and table_name like 'growth_observation%' order by table_name` + ); + expect(result.rows.map((r) => r.name)).toContain('growth_observations'); + expect(result.rows.map((r) => r.name)).toContain('growth_observation_work'); + const privateColumns = await db.execute( + `select column_name from information_schema.columns where table_name in ('growth_observation_source_health_v1','growth_observation_subject_overview_v1','growth_observation_work_health_v1') and column_name in ('email_normalized','git_display_name','identity_digest')` + ); + expect(privateColumns.rows).toEqual([]); + }); + it('enforces uniqueness, trust, lease shape and cascading fixture removal', async () => { + const now = new Date(), + batch = evidenceFixture(now); + try { + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const row = ( + await db.execute<{ id: string; subject_id: string }>( + 'select id,subject_id from growth_observations where event_id=$1', + [batch.events[0].eventId] + ) + ).rows[0]; + await expect( + db.execute( + "insert into growth_observation_subjects(namespace,external_id,first_received_at,last_received_at) values('installation',$1,$2,$2)", + [batch.events[0].subject.id, now] + ) + ).rejects.toMatchObject({ code: '23505' }); + await expect( + db.execute( + "update growth_observations set trust='verified' where id=$1", + [row.id] + ) + ).rejects.toMatchObject({ code: '23514' }); + await expect( + db.execute( + "update growth_observation_work set status='leased' where observation_id=$1", + [row.id] + ) + ).rejects.toMatchObject({ code: '23514' }); + await expect( + db.execute( + 'insert into growth_observation_work(observation_id,available_at,updated_at) values($1,$2,$2)', + [randomUUID(), now] + ) + ).rejects.toMatchObject({ code: '23503' }); + await db.execute('delete from growth_observation_subjects where id=$1', [ + row.subject_id, + ]); + for (const table of [ + 'growth_observations', + 'growth_observation_identities', + 'growth_observation_work', + ]) { + const column = + table === 'growth_observations' ? 'id' : 'observation_id'; + expect( + ( + await db.execute(`select 1 from ${table} where ${column}=$1`, [ + row.id, + ]) + ).rows + ).toHaveLength(0); + } + } finally { + await cleanEvidence(db, [batch.events[0].subject.id]); + } + }); +}); diff --git a/libs/growth/vite.integration.config.mts b/libs/growth/vite.integration.config.mts index ec66a6bdf..afd2c421a 100644 --- a/libs/growth/vite.integration.config.mts +++ b/libs/growth/vite.integration.config.mts @@ -27,6 +27,9 @@ export default defineConfig({ test: { environment: 'node', fileParallelism: false, + // Real Neon transactions include network round trips and cold compute startup. + testTimeout: 30_000, + hookTimeout: 30_000, globals: true, include: ['libs/growth/test/**/*.integration.spec.ts'], }, diff --git a/libs/growth/vite.operator-cli.config.mts b/libs/growth/vite.operator-cli.config.mts index 9c1b50169..dd7b77c6e 100644 --- a/libs/growth/vite.operator-cli.config.mts +++ b/libs/growth/vite.operator-cli.config.mts @@ -15,6 +15,7 @@ export default defineConfig({ include: [ 'scripts/apply-migrations.spec.ts', 'scripts/growth-control.spec.ts', + 'scripts/growth-observability.spec.ts', 'scripts/import-resend-lifecycle.spec.ts', 'scripts/cancel-resend-lifecycle.spec.ts', ], diff --git a/migrations/0004_growth_observability.sql b/migrations/0004_growth_observability.sql new file mode 100644 index 000000000..b6bd35363 --- /dev/null +++ b/migrations/0004_growth_observability.sql @@ -0,0 +1,93 @@ +CREATE TABLE growth_observation_subjects ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + namespace text NOT NULL CHECK (namespace IN ('website_session','installation','development_browser')), + external_id uuid NOT NULL, + first_received_at timestamptz NOT NULL, + last_received_at timestamptz NOT NULL, + UNIQUE (namespace, external_id) +); +CREATE TABLE growth_observations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + source text NOT NULL CHECK (source IN ('website','install','runtime')), + event_id uuid NOT NULL, + subject_id uuid NOT NULL REFERENCES growth_observation_subjects(id) ON DELETE CASCADE, + session_id uuid, + kind text NOT NULL CHECK (length(kind) BETWEEN 1 AND 100), + schema_version smallint NOT NULL CHECK (schema_version=1), + collector_version text NOT NULL CHECK (length(collector_version) BETWEEN 1 AND 64), + identity_scope text NOT NULL CHECK (identity_scope IN ('persistent','session','memory')), + occurred_at timestamptz NOT NULL, + received_at timestamptz NOT NULL, + trust text NOT NULL DEFAULT 'client_reported' CHECK (trust='client_reported'), + properties jsonb NOT NULL CHECK (jsonb_typeof(properties)='object'), + public_digest text NOT NULL CHECK (length(public_digest)=64), + identity_digest text CHECK (length(identity_digest)=64), + identity_digest_key_version smallint CHECK (identity_digest_key_version>0), + redacted_at timestamptz, + UNIQUE(source,event_id), + CHECK ((identity_digest IS NULL)=(identity_digest_key_version IS NULL)) +); +CREATE INDEX growth_observations_subject_time ON growth_observations(subject_id,received_at,id); +CREATE INDEX growth_observations_source_time ON growth_observations(source,received_at,id); +CREATE TABLE growth_observation_identities ( + observation_id uuid PRIMARY KEY REFERENCES growth_observations(id) ON DELETE CASCADE, + email_normalized text CHECK (length(email_normalized) BETWEEN 1 AND 320), + git_display_name text CHECK (length(git_display_name) BETWEEN 1 AND 160), + git_config_origin text CHECK (git_config_origin IN ('local','global','unknown')), + repository_provider text CHECK (repository_provider IN ('github','gitlab','bitbucket')), + repository_owner text CHECK (length(repository_owner) BETWEEN 1 AND 100), + email_lookup_hmac text, + email_key_version smallint CHECK (email_key_version>0), + CHECK ((email_normalized IS NULL)=(email_lookup_hmac IS NULL)), + CHECK ((email_normalized IS NULL)=(email_key_version IS NULL)), + CHECK ((repository_provider IS NULL)=(repository_owner IS NULL)) +); +CREATE INDEX growth_observation_identities_lookup ON growth_observation_identities(email_key_version,email_lookup_hmac); +CREATE INDEX growth_observation_identities_email ON growth_observation_identities(email_normalized); +CREATE TABLE growth_observation_redactions ( + selector_kind text NOT NULL CHECK (selector_kind IN ('subject','email')), + selector_key text NOT NULL, + key_version smallint NOT NULL, + redacted_at timestamptz NOT NULL, + PRIMARY KEY(selector_kind,selector_key,key_version), + CHECK ((selector_kind='subject' AND key_version=0) OR (selector_kind='email' AND key_version>0)) +); +CREATE TABLE growth_observation_work ( + observation_id uuid PRIMARY KEY REFERENCES growth_observations(id) ON DELETE CASCADE, + generation bigint NOT NULL DEFAULT 1 CHECK (generation>0), + projection_version text NOT NULL DEFAULT 'observation-facts-v1', + status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','leased','completed','failed')), + available_at timestamptz NOT NULL, + lease_token uuid, + lease_until timestamptz, + attempts integer NOT NULL DEFAULT 0 CHECK (attempts>=0), + last_error_code text CHECK (length(last_error_code)<=80), + updated_at timestamptz NOT NULL, + CHECK ((status='leased' AND lease_token IS NOT NULL AND lease_until IS NOT NULL) + OR (status<>'leased' AND lease_token IS NULL AND lease_until IS NULL)) +); +CREATE INDEX growth_observation_work_due ON growth_observation_work(available_at,observation_id) WHERE status IN ('pending','leased'); +CREATE TABLE growth_observation_facts ( + observation_id uuid PRIMARY KEY REFERENCES growth_observations(id) ON DELETE CASCADE, + generation bigint NOT NULL, + projection_version text NOT NULL, + projected_at timestamptz NOT NULL, + active_day date NOT NULL, + milestone_kind text CHECK (milestone_kind IN ('transport.connected','runtime.first_stream_completed','thread.persisted','interrupt.handled','generative_ui.rendered')), + source text NOT NULL CHECK (source IN ('website','install','runtime')), + subject_id uuid NOT NULL REFERENCES growth_observation_subjects(id) ON DELETE CASCADE +); +CREATE TABLE growth_collection_budgets ( + bucket_key text NOT NULL, + window_start timestamptz NOT NULL, + count bigint NOT NULL CHECK (count>0), + PRIMARY KEY(bucket_key,window_start) +); +CREATE TABLE growth_observation_operations ( + operation_id uuid PRIMARY KEY, + kind text NOT NULL CHECK (kind IN ('replay','redact')), + requested_at timestamptz NOT NULL, + selection_digest text NOT NULL, + selected_count integer NOT NULL CHECK (selected_count>=0), + completed_at timestamptz NOT NULL +); diff --git a/migrations/0005_growth_observability_views.sql b/migrations/0005_growth_observability_views.sql new file mode 100644 index 000000000..4dc505a77 --- /dev/null +++ b/migrations/0005_growth_observability_views.sql @@ -0,0 +1,30 @@ +CREATE VIEW growth_observation_source_health_v1 AS +SELECT source, kind, collector_version, + CASE WHEN source='install' THEN properties->>'environment' END AS environment, + count(*) AS observation_count, + count(DISTINCT subject_id) AS subject_count, + count(DISTINCT (subject_id,session_id)) FILTER (WHERE session_id IS NOT NULL) AS session_count, + max(received_at) AS last_received_at +FROM growth_observations +GROUP BY source, kind, collector_version, CASE WHEN source='install' THEN properties->>'environment' END; + +CREATE VIEW growth_observation_subject_overview_v1 AS +SELECT s.id AS subject_id,s.namespace,s.first_received_at,s.last_received_at, + count(o.id) AS observation_count, + count(o.id) FILTER (WHERE f.observation_id IS NULL) AS unprojected_count, + count(DISTINCT o.session_id) AS session_count, + count(DISTINCT f.active_day) AS active_days, + count(DISTINCT (f.milestone_kind,o.properties->>'integration')) FILTER (WHERE f.milestone_kind IS NOT NULL) AS attained_milestone_count +FROM growth_observation_subjects s +LEFT JOIN growth_observations o ON o.subject_id=s.id +LEFT JOIN growth_observation_facts f ON f.observation_id=o.id +GROUP BY s.id; + +CREATE VIEW growth_observation_work_health_v1 AS +SELECT status,projection_version,count(*) AS work_count, + count(*) FILTER (WHERE status='pending' AND available_at<=now()) AS due_count, + count(*) FILTER (WHERE status='leased' AND lease_until<=now()) AS expired_lease_count, + min(o.received_at) FILTER (WHERE status<>'completed') AS oldest_unprocessed_receipt, + max(attempts) AS maximum_attempts +FROM growth_observation_work w JOIN growth_observations o ON o.id=w.observation_id +GROUP BY status,projection_version; diff --git a/migrations/0006_growth_form_observations.sql b/migrations/0006_growth_form_observations.sql new file mode 100644 index 000000000..2f66685bb --- /dev/null +++ b/migrations/0006_growth_form_observations.sql @@ -0,0 +1,21 @@ +ALTER TABLE growth_observation_subjects DROP CONSTRAINT growth_observation_subjects_namespace_check; +ALTER TABLE growth_observation_subjects ADD CONSTRAINT growth_observation_subjects_namespace_check + CHECK (namespace IN ('website_session','installation','development_browser','form_submission')); +ALTER TABLE growth_observations DROP CONSTRAINT growth_observations_source_check; +ALTER TABLE growth_observations ADD CONSTRAINT growth_observations_source_check CHECK (source IN ('website','install','runtime','form')); +ALTER TABLE growth_observations DROP CONSTRAINT growth_observations_trust_check; +ALTER TABLE growth_observations ADD CONSTRAINT growth_observations_trust_check CHECK ( + (source='form' AND trust='server_verified' AND kind='form.accepted') OR + (source IN ('website','install','runtime') AND trust='client_reported' AND kind<>'form.accepted') +); +ALTER TABLE growth_observation_facts DROP CONSTRAINT growth_observation_facts_source_check; +ALTER TABLE growth_observation_facts ADD CONSTRAINT growth_observation_facts_source_check CHECK (source IN ('website','install','runtime','form')); + +CREATE TABLE growth_observation_form_links ( + observation_id uuid PRIMARY KEY REFERENCES growth_observations(id) ON DELETE CASCADE, + activity_id bigint NOT NULL UNIQUE REFERENCES growth_activity(id) ON DELETE CASCADE, + contact_id uuid NOT NULL REFERENCES growth_contacts(id) ON DELETE CASCADE +); +CREATE INDEX growth_observation_form_links_contact ON growth_observation_form_links(contact_id); +CREATE INDEX growth_activity_form_observation_candidates ON growth_activity(id) + WHERE kind='contact.form_submission' AND event_key LIKE 'form:%:accepted'; diff --git a/migrations/0007_growth_install_runtime.sql b/migrations/0007_growth_install_runtime.sql new file mode 100644 index 000000000..374e7482f --- /dev/null +++ b/migrations/0007_growth_install_runtime.sql @@ -0,0 +1,15 @@ +alter table growth_observations add column installation_token_digest text; +alter table growth_observations add constraint growth_installation_token_digest_format + check (installation_token_digest is null or installation_token_digest ~ '^[0-9a-f]{64}$'); +create index growth_observation_installation_token on growth_observations(installation_token_digest,source) + where installation_token_digest is not null; + +create table growth_install_runtime_links ( + runtime_observation_id uuid primary key references growth_observations(id) on delete cascade, + install_observation_id uuid references growth_observations(id) on delete cascade, + contact_id uuid references growth_contacts(id) on delete set null, + outcome text not null check (outcome in ('approved','ineligible','conflicted')), + evaluated_at timestamptz not null +); +create index growth_install_runtime_contact on growth_install_runtime_links(contact_id) + where contact_id is not null; diff --git a/package.json b/package.json index 83167c0a4..c6ead4408 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "generate-whitepaper": "npx tsx apps/website/scripts/generate-whitepaper.ts", "db:migrate": "tsx scripts/apply-migrations.mts", "growth:control": "tsx scripts/growth-control.mts", + "growth:observability": "tsx scripts/growth-observability.mts", "growth:cancel-resend": "tsx scripts/cancel-resend-lifecycle.mts", "growth:import-resend": "tsx scripts/import-resend-lifecycle.mts", "marketing:channels:x:auth": "tsx --env-file=.env marketing/channels/src/x/auth-cli.ts", diff --git a/scripts/growth-observability.mts b/scripts/growth-observability.mts new file mode 100644 index 000000000..0f07ebd68 --- /dev/null +++ b/scripts/growth-observability.mts @@ -0,0 +1,257 @@ +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + createDatabaseExecutor, + readObservationHealth, + readTimeline, + readObservationIdentity, + processObservations, + projectFormObservations, + replayObservations, + redactObservationEvidence, + initializeObservationRedactions, + ObservationError, + type SqlExecutor, + type EmailHmacKeyring, + collectionSource, +} from '../libs/growth/src/index.ts'; +import { uuid } from '../libs/growth/src/lib/observability/contracts.ts'; +import { parseEmailHmacKeyringEnvironment } from './growth-control.mts'; + +export interface ObservabilityOperatorDependencies { + createDatabase(): SqlExecutor; + loadKeyring(): EmailHmacKeyring; + environment(): Readonly>; + now(): Date; + readEmail(): Promise; + writeOutput(value: string): void; + writeError(value: string): void; +} +const flags: Record = { + health: ['from', 'to'], + timeline: ['subject', 'cursor', 'limit'], + detail: ['observation', 'include-identity'], + process: ['limit'], + 'project-forms': ['limit'], + replay: ['subject', 'source', 'from', 'to', 'operation', 'max-events'], + redact: ['subject', 'email-stdin', 'operation'], + 'initialize-redactions': ['limit', 'cursor'], +}; +function invalid(): never { + throw new ObservationError('invalid_arguments'); +} +function date(value: string | undefined): Date { + if (!value) invalid(); + const result = new Date(value); + if (!Number.isFinite(result.getTime())) invalid(); + return result; +} +function count( + value: string | undefined, + maximum: number, + fallback?: number +): number { + if (value === undefined && fallback !== undefined) return fallback; + if (!value || !/^\d+$/u.test(value)) invalid(); + const n = Number(value); + if (!Number.isInteger(n) || n < 1 || n > maximum) invalid(); + return n; +} +function parseArguments(argv: readonly string[]) { + const command = argv[0]; + if (!command || !Object.hasOwn(flags, command)) invalid(); + const args: Record = {}; + for (let i = 1; i < argv.length; i++) { + const key = argv[i].slice(2); + if ( + !argv[i].startsWith('--') || + !flags[command].includes(key) || + Object.hasOwn(args, key) + ) + invalid(); + if (key === 'include-identity' || key === 'email-stdin') { + args[key] = 'true'; + continue; + } + const value = argv[++i]; + if (!value || value.startsWith('--')) invalid(); + args[key] = value; + } + if (command === 'health') { + const from = date(args.from), + to = date(args.to); + if (to <= from || to.getTime() - from.getTime() > 31 * 86400000) invalid(); + } + if (command === 'timeline') { + uuid(args.subject); + count(args.limit, 100, 100); + if (args.cursor && args.cursor.length > 300) invalid(); + } + if (command === 'detail') { + uuid(args.observation); + if (args['include-identity'] !== 'true') invalid(); + } + if (command === 'process' || command === 'project-forms') + count(args.limit, 100); + if (command === 'initialize-redactions') { + count(args.limit, 100); + if (args.cursor) uuid(args.cursor); + } + if (command === 'redact') { + uuid(args.operation); + if (Boolean(args.subject) === Boolean(args['email-stdin'])) invalid(); + if (args.subject) uuid(args.subject); + } + if (command === 'replay') { + uuid(args.operation); + count(args['max-events'], 1000); + if (args.subject) { + uuid(args.subject); + if (args.source || args.from || args.to) invalid(); + } else { + collectionSource(args.source); + const from = date(args.from), + to = date(args.to); + if (to <= from || to.getTime() - from.getTime() > 86400000) invalid(); + } + } + return { command, args }; +} +export async function runGrowthObservability( + argv: readonly string[], + deps: ObservabilityOperatorDependencies +): Promise { + let parsed: ReturnType; + try { + parsed = parseArguments(argv); + } catch { + deps.writeError('invalid_arguments'); + return 2; + } + const { command, args } = parsed; + let db: SqlExecutor | undefined; + try { + if ( + (command === 'process' || command === 'project-forms') && + deps.environment()['GROWTH_OBSERVATION_PROCESSING_ENABLED'] !== 'true' + ) { + deps.writeOutput(JSON.stringify({ command, disabled: true })); + return 0; + } + let email: string | undefined; + if (args['email-stdin']) { + email = (await deps.readEmail()).trim(); + if (!email || email.length > 320 || /[\r\n]/u.test(email)) invalid(); + } + db = deps.createDatabase(); + let result: unknown; + switch (command) { + case 'project-forms': + result = await projectFormObservations(db, { + enabled: true, + limit: count(args.limit, 100), + now: deps.now, + }); + break; + case 'health': + result = await readObservationHealth(db, { + from: date(args.from), + to: date(args.to), + }); + break; + case 'timeline': + result = await readTimeline(db, args.subject, { + limit: count(args.limit, 100, 100), + cursor: args.cursor, + }); + break; + case 'detail': + result = await readObservationIdentity(db, args.observation); + break; + case 'process': + result = await processObservations(db, { + enabled: true, + limit: count(args.limit, 100), + now: deps.now, + }); + break; + case 'replay': + result = await replayObservations( + db, + { + operationId: args.operation, + maxEvents: count(args['max-events'], 1000), + ...(args.subject + ? { subjectId: args.subject } + : { + source: collectionSource(args.source), + from: date(args.from), + to: date(args.to), + }), + }, + deps.now() + ); + break; + case 'redact': + result = await redactObservationEvidence( + db, + args.subject ? { subjectId: args.subject } : { email: email! }, + { + operationId: args.operation, + now: deps.now(), + keyring: deps.loadKeyring(), + } + ); + break; + case 'initialize-redactions': + result = await initializeObservationRedactions( + db, + { limit: count(args.limit, 100), cursor: args.cursor }, + deps.now() + ); + break; + } + deps.writeOutput(JSON.stringify({ command, result })); + return 0; + } catch (error) { + deps.writeError( + error instanceof ObservationError ? error.code : 'operation_failed' + ); + return 1; + } finally { + if (db?.close) await db.close().catch(() => undefined); + } +} +async function readEmail(): Promise { + const chunks: Buffer[] = []; + let length = 0; + for await (const chunk of process.stdin) { + const bytes = Buffer.from(chunk); + length += bytes.length; + if (length > 1280) throw new ObservationError('invalid_arguments'); + chunks.push(bytes); + } + return Buffer.concat(chunks).toString('utf8'); +} +if ( + process.argv[1] && + pathToFileURL(resolve(process.argv[1])).href === import.meta.url +) { + void runGrowthObservability(process.argv.slice(2), { + createDatabase: () => createDatabaseExecutor(), + loadKeyring: () => parseEmailHmacKeyringEnvironment(process.env), + environment: () => process.env, + now: () => new Date(), + readEmail, + writeOutput: (line) => process.stdout.write(line + '\n'), + writeError: (line) => process.stderr.write(line + '\n'), + }).then( + (code) => { + process.exitCode = code; + }, + () => { + process.stderr.write('operation_failed\n'); + process.exitCode = 1; + } + ); +} diff --git a/scripts/growth-observability.spec.ts b/scripts/growth-observability.spec.ts new file mode 100644 index 000000000..03484a845 --- /dev/null +++ b/scripts/growth-observability.spec.ts @@ -0,0 +1,109 @@ +import { runGrowthObservability } from './growth-observability.mts'; + +function dependencies() { + return { + createDatabase: vi.fn(() => { + throw new Error('must not connect'); + }), + loadKeyring: vi.fn(() => ({ + active: { version: 1, secret: 'x'.repeat(32) }, + })), + environment: () => ({}), + now: () => new Date('2026-09-04T12:00:00Z'), + readEmail: vi.fn(async () => ''), + writeOutput: vi.fn(), + writeError: vi.fn(), + }; +} +describe('observation operator boundary', () => { + it('allows redacted reads with processing disabled and closes the executor', async () => { + const execute = vi.fn(async () => ({ rows: [] })), + close = vi.fn(async () => undefined); + const deps = { + ...dependencies(), + createDatabase: vi.fn(() => ({ execute, close, transaction: vi.fn() })), + }; + expect( + await runGrowthObservability( + [ + 'health', + '--from', + '2026-09-04T00:00:00Z', + '--to', + '2026-09-05T00:00:00Z', + ], + deps + ) + ).toBe(0); + expect(close).toHaveBeenCalledOnce(); + expect(deps.loadKeyring).not.toHaveBeenCalled(); + expect(deps.writeOutput).toHaveBeenCalledWith( + expect.stringContaining('currentQueue') + ); + }); + it('closes after a query failure and returns only a safe error', async () => { + const execute = vi.fn(async () => { + throw new Error('private@example.invalid'); + }), + close = vi.fn(async () => undefined); + const deps = { + ...dependencies(), + createDatabase: vi.fn(() => ({ execute, close, transaction: vi.fn() })), + }; + expect( + await runGrowthObservability( + ['timeline', '--subject', '11111111-1111-4111-8111-111111111111'], + deps + ) + ).toBe(1); + expect(close).toHaveBeenCalledOnce(); + expect(deps.writeError).toHaveBeenCalledWith('operation_failed'); + expect(deps.writeOutput).not.toHaveBeenCalled(); + }); + it.each([ + ['detail', '--observation', '11111111-1111-4111-8111-111111111111'], + ['redact', '--email', 'private@example.invalid'], + ['timeline', '--subject', 'invalid'], + ['process', '--limit', '100000'], + ['replay', '--subject', '11111111-1111-4111-8111-111111111111'], + ])( + 'rejects invalid command arguments before database access: %s', + async (...args) => { + const deps = dependencies(); + expect(await runGrowthObservability(args, deps)).toBe(2); + expect(deps.createDatabase).not.toHaveBeenCalled(); + expect(JSON.stringify(deps.writeError.mock.calls)).not.toContain( + 'private@example.invalid' + ); + } + ); + it.each(['process', 'project-forms'])( + 'does not run %s when the independent switch is off', + async (command) => { + const deps = dependencies(); + expect( + await runGrowthObservability([command, '--limit', '10'], deps) + ).toBe(0); + expect(deps.createDatabase).not.toHaveBeenCalled(); + expect(deps.writeOutput).toHaveBeenCalledWith( + expect.stringContaining('disabled') + ); + } + ); + it('does not echo connection exceptions', async () => { + const deps = dependencies(); + expect( + await runGrowthObservability( + [ + 'health', + '--from', + '2026-09-04T00:00:00Z', + '--to', + '2026-09-05T00:00:00Z', + ], + deps + ) + ).toBe(1); + expect(deps.writeError).toHaveBeenCalledWith('operation_failed'); + }); +}); From 43cbc2129b30c46e5b7a3432ee195f365756b155 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Fri, 4 Sep 2026 21:13:20 -0700 Subject: [PATCH 2/2] test(growth): include observability CLI in CI scope coverage --- scripts/ci-scope.spec.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/ci-scope.spec.mjs b/scripts/ci-scope.spec.mjs index aa9efb23c..16f79d64d 100644 --- a/scripts/ci-scope.spec.mjs +++ b/scripts/ci-scope.spec.mjs @@ -174,6 +174,7 @@ describe('growth lifecycle project ownership', () => { ['tools/google-mailbox-poller/Code.gs', 'google-mailbox-poller'], ['scripts/apply-migrations.mts', 'growth'], ['scripts/growth-control.mts', 'growth'], + ['scripts/growth-observability.mts', 'growth'], ['scripts/import-resend-lifecycle.mts', 'growth'], ['migrations/0001_rate_limit_events.sql', 'growth'], ['migrations/0002_growth_control_plane.sql', 'growth'], @@ -185,7 +186,7 @@ describe('growth lifecycle project ownership', () => { }); } - it('runs exactly the four database/operator CLI suites in its dedicated target', async () => { + it('runs the database/operator CLI suites in its dedicated target', async () => { const project = JSON.parse( await readFile('libs/growth/project.json', 'utf8') ); @@ -198,6 +199,7 @@ describe('growth lifecycle project ownership', () => { 'scripts/apply-migrations.spec.ts', 'scripts/cancel-resend-lifecycle.spec.ts', 'scripts/growth-control.spec.ts', + 'scripts/growth-observability.spec.ts', 'scripts/import-resend-lifecycle.spec.ts', ]); });