diff --git a/apps/lifecycle/README.md b/apps/lifecycle/README.md index 86e4e38bc..10f98e856 100644 --- a/apps/lifecycle/README.md +++ b/apps/lifecycle/README.md @@ -17,6 +17,10 @@ Deploy backend observation acceptance and bridge resolution with the rollout swi All three campaign steps are founder session offers and send without waiting for an enrichment artifact; a cited research angle only selects an angle-flavored version of the same offer. A persisted `install_runtime` enrollment reason keeps all three steps generic even if optional research later becomes available. Form and project-claim enrollments retain their existing behavior. The shared delivery authorization, reply/suppression stops, mailbox recovery guard, unsubscribe links, and once-per-contact three-step enrollment remain in force; install-derived eligibility does not verify identity or employment. +An eligible install/runtime link also queues at most one optional company-enrichment job per contact when the admitted install email has a valid non-personal domain. It uses the existing enrichment worker, capture provider and artifact schema. Its payload contains observation references and an explicit `install_runtime` source; it does not invent a form submission or verified company association. The worker rechecks contact approval, stops, lease, and linked evidence before capture and again before model execution. Persistence checks these controls again, and evidence redaction cancels affected work and removes retained artifacts. A skipped or failed enrichment job does not delay the generic hello sequence. + +Use the [growth operator reports](../../libs/growth/README.md) for the bounded funnel and contact journey. Collection and activation can precede fact projection; the report exposes pending observation work. Projection currently runs through the existing operator command rather than the lifecycle tick. + Recipient delivery also requires `GROWTH_PUBLIC_ACTION_ORIGIN`, a server-only bare HTTPS origin for the Website deployment that owns `/api/unsubscribe`. In preview, use a dedicated public custom-domain alias for the exact Website preview deployment while keeping generated preview URLs protected; the signed action token is the application-layer authorization. In production, use the canonical Website origin. Paths, query strings, fragments, credentials, and HTTP origins are rejected. The lifecycle service uses this value only to construct opaque, contact-bound unsubscribe action URLs; it never derives the origin from a request or hardcodes the production site. Set `GROWTH_DATABASE_ENVIRONMENT` to exactly `preview`, `production`, or `test` in every process that handles verified Resend events. A verified webhook whose `environment` provider tag is missing or differs from that value is acknowledged without opening a growth transaction or changing delivery/suppression state. diff --git a/apps/lifecycle/src/campaign/send.spec.ts b/apps/lifecycle/src/campaign/send.spec.ts index 4cb7833ee..a92f0d6ab 100644 --- a/apps/lifecycle/src/campaign/send.spec.ts +++ b/apps/lifecycle/src/campaign/send.spec.ts @@ -1,5 +1,6 @@ import { createUnsubscribeActionUrl, + JobLeaseConflictError, unsubscribeActionUrlValue, type GrowthArtifact, type GrowthJob, @@ -392,6 +393,9 @@ function dependencies( return { now: () => NOW, readJobContext: vi.fn().mockResolvedValue(context()), + readInstallRuntimeEnrichmentContext: vi + .fn() + .mockResolvedValue({ companyDomain: 'example.com' }), createUnsubscribeUrl: vi.fn(() => UNSUBSCRIBE), sendRecipient: vi.fn().mockResolvedValue({ accepted: true, @@ -641,6 +645,120 @@ describe('dispatchLifecycleAppOwnedJob', () => { expect(deps.completeJob).toHaveBeenCalledOnce(); }); + it('enriches an admitted install domain without inventing a form submission', async () => { + const deps = dependencies({ + readJobContext: vi + .fn() + .mockResolvedValue( + context({ + formSubmission: {}, + companyDomain: null, + emailClassification: 'unknown', + }) + ), + readInstallRuntimeEnrichmentContext: vi + .fn() + .mockResolvedValue({ companyDomain: 'neon.tech' }), + }); + await expect( + dispatchLifecycleAppOwnedJob( + {} as SqlExecutor, + job('enrich', { source: 'install_runtime' }), + {}, + deps + ) + ).resolves.toBe('completed'); + expect(deps.fetchCompanyEvidence).toHaveBeenCalledWith( + 'neon.tech', + expect.any(AbortSignal) + ); + expect(deps.generateArtifact).toHaveBeenCalledWith( + expect.objectContaining({ + formFacts: { source: 'install_runtime', companyDomain: 'neon.tech' }, + }), + expect.any(AbortSignal) + ); + expect(deps.readInstallRuntimeEnrichmentContext).toHaveBeenCalledTimes(2); + expect(deps.sendRecipient).not.toHaveBeenCalled(); + }); + + it('cancels install enrichment if current evidence or contact eligibility is unavailable', async () => { + const deps = dependencies({ + readInstallRuntimeEnrichmentContext: vi.fn().mockResolvedValue(null), + }); + await expect( + dispatchLifecycleAppOwnedJob( + {} as SqlExecutor, + job('enrich', { source: 'install_runtime' }), + {}, + deps + ) + ).resolves.toBe('cancelled'); + expect(deps.fetchCompanyEvidence).not.toHaveBeenCalled(); + expect(deps.generateArtifact).not.toHaveBeenCalled(); + expect(deps.persistArtifact).not.toHaveBeenCalled(); + }); + + it('rechecks install eligibility after capture before making a model call', async () => { + const deps = dependencies({ + readInstallRuntimeEnrichmentContext: vi + .fn() + .mockResolvedValueOnce({ companyDomain: 'neon.tech' }) + .mockResolvedValueOnce(null), + }); + await expect( + dispatchLifecycleAppOwnedJob( + {} as SqlExecutor, + job('enrich', { source: 'install_runtime' }), + {}, + deps + ) + ).resolves.toBe('cancelled'); + expect(deps.fetchCompanyEvidence).toHaveBeenCalledOnce(); + expect(deps.generateArtifact).not.toHaveBeenCalled(); + expect(deps.persistArtifact).not.toHaveBeenCalled(); + }); + + it('abandons install enrichment when a stop has already revoked its lease', async () => { + const deps = dependencies({ + readInstallRuntimeEnrichmentContext: vi + .fn() + .mockResolvedValueOnce({ companyDomain: 'neon.tech' }) + .mockResolvedValueOnce(null), + cancelJob: vi.fn().mockRejectedValue(new JobLeaseConflictError(job().id)), + }); + await expect( + dispatchLifecycleAppOwnedJob( + {} as SqlExecutor, + job('enrich', { source: 'install_runtime' }), + {}, + deps + ) + ).resolves.toBe('cancelled'); + expect(deps.cancelJob).toHaveBeenCalledOnce(); + expect(deps.deferJob).not.toHaveBeenCalled(); + expect(deps.failJob).not.toHaveBeenCalled(); + expect(deps.generateArtifact).not.toHaveBeenCalled(); + expect(deps.persistArtifact).not.toHaveBeenCalled(); + }); + + it('keeps ordinary database cancellation failures on the enrichment retry path', async () => { + const deps = dependencies({ + readInstallRuntimeEnrichmentContext: vi.fn().mockResolvedValue(null), + cancelJob: vi.fn().mockRejectedValue(new Error('database unavailable')), + }); + await expect( + dispatchLifecycleAppOwnedJob( + {} as SqlExecutor, + job('enrich', { source: 'install_runtime' }), + {}, + deps + ) + ).resolves.toBe('deferred'); + expect(deps.deferJob).toHaveBeenCalledOnce(); + expect(deps.generateArtifact).not.toHaveBeenCalled(); + }); + it('does not fetch company pages for the personal-email neutral path', async () => { const deps = dependencies({ readJobContext: vi.fn().mockResolvedValue( diff --git a/apps/lifecycle/src/campaign/send.ts b/apps/lifecycle/src/campaign/send.ts index 353cd968a..830809ff6 100644 --- a/apps/lifecycle/src/campaign/send.ts +++ b/apps/lifecycle/src/campaign/send.ts @@ -9,6 +9,7 @@ import { createUnsubscribeActionUrl, deferLeasedJob, failLeasedJob, + JobLeaseConflictError, loadGrowthTokenKeyring, markProviderAcceptanceUnknown, markInternalNotificationUnknown, @@ -17,6 +18,7 @@ import { normalizeRecipientEmail, persistJobArtifact, readLifecycleJobContext, + readInstallRuntimeEnrichmentContext, recordProviderAcceptance, RECIPIENT_EMAIL_SENDER, recomputeContactScore, @@ -92,6 +94,7 @@ interface DeferLeasedJobInput extends LeasedTransitionInput { } export interface LifecycleJobDependencies { + readInstallRuntimeEnrichmentContext: typeof readInstallRuntimeEnrichmentContext; now: () => Date; readJobContext: ( executor: SqlExecutor, @@ -552,17 +555,35 @@ export async function dispatchLifecycleAppOwnedJob( if (job.kind === 'enrich') { try { + const installRuntime = job.payload['source'] === 'install_runtime'; + const installContext = installRuntime + ? await dependencies.readInstallRuntimeEnrichmentContext(executor, { + jobId: job.id, + leaseToken, + now: dependencies.now(), + }) + : null; + if (installRuntime && !installContext) { + await dependencies.cancelJob(executor, { + jobId: job.id, + leaseToken, + now: dependencies.now(), + errorCode: 'install_runtime_evidence_unavailable', + }); + return 'cancelled'; + } + const companyDomain = installRuntime + ? installContext?.companyDomain + : context.companyDomain; const deterministicScore = await dependencies.readDeterministicScore( executor, context.contactId ); signal.throwIfAborted(); const companyPages = - context.emailClassification !== 'personal' && context.companyDomain - ? await dependencies.fetchCompanyEvidence( - context.companyDomain, - signal - ) + (installRuntime || context.emailClassification !== 'personal') && + companyDomain + ? await dependencies.fetchCompanyEvidence(companyDomain, signal) : []; signal.throwIfAborted(); const paper = context.formSubmission['paper']; @@ -571,40 +592,68 @@ export async function dispatchLifecycleAppOwnedJob( const timeline = context.formSubmission['timeline']; const researchInput = buildResearchInput({ formFacts: { - source: formSource(context.formSubmission['form_kind']), - emailClassification: context.emailClassification, - ...(context.displayName ? { displayName: context.displayName } : {}), - ...(context.companyName ? { companyName: context.companyName } : {}), - ...(context.companyDomain - ? { companyDomain: context.companyDomain } - : {}), - ...(paper === 'overview' || - paper === 'angular' || - paper === 'render' || - paper === 'chat' - ? { paper } - : {}), - ...(pilotInterest === 'yes' || - pilotInterest === 'maybe' || - pilotInterest === 'no' - ? { pilotInterest } - : {}), - ...(teamSize === '1-5' || - teamSize === '6-25' || - teamSize === '26-100' || - teamSize === '100+' - ? { teamSize } - : {}), - ...(timeline === 'this_quarter' || - timeline === 'next_quarter' || - timeline === '6_plus_months' || - timeline === 'exploring' - ? { timeline } - : {}), + ...(installRuntime + ? { + source: 'install_runtime', + emailClassification: 'unknown', + companyDomain, + } + : { + source: formSource(context.formSubmission['form_kind']), + emailClassification: context.emailClassification, + ...(context.displayName + ? { displayName: context.displayName } + : {}), + ...(context.companyName + ? { companyName: context.companyName } + : {}), + ...(context.companyDomain + ? { companyDomain: context.companyDomain } + : {}), + ...(paper === 'overview' || + paper === 'angular' || + paper === 'render' || + paper === 'chat' + ? { paper } + : {}), + ...(pilotInterest === 'yes' || + pilotInterest === 'maybe' || + pilotInterest === 'no' + ? { pilotInterest } + : {}), + ...(teamSize === '1-5' || + teamSize === '6-25' || + teamSize === '26-100' || + teamSize === '100+' + ? { teamSize } + : {}), + ...(timeline === 'this_quarter' || + timeline === 'next_quarter' || + timeline === '6_plus_months' || + timeline === 'exploring' + ? { timeline } + : {}), + }), }, deterministicScore, companyPages, }); + if (installRuntime) { + const current = await dependencies.readInstallRuntimeEnrichmentContext( + executor, + { jobId: job.id, leaseToken, now: dependencies.now() } + ); + signal.throwIfAborted(); + if (!current || current.companyDomain !== companyDomain) { + await dependencies.cancelJob(executor, { + jobId: job.id, + leaseToken, + now: dependencies.now(), + errorCode: 'install_runtime_evidence_unavailable', + }); + return 'cancelled'; + } + } const artifact = await dependencies.generateArtifact( researchInput, signal @@ -629,6 +678,14 @@ export async function dispatchLifecycleAppOwnedJob( } catch (error) { signal.throwIfAborted(); if (error instanceof DeterministicLifecycleJobError) throw error; + if ( + job.payload['source'] === 'install_runtime' && + error instanceof JobLeaseConflictError + ) { + // Stop/redaction or another worker already owns the durable state. + // Cancel this dispatch without attempting another transition on its revoked lease. + return 'cancelled'; + } if (job.attempts < 2) { const retryAt = dependencies.now(); await dependencies.deferJob(executor, { @@ -899,6 +956,7 @@ export function createDefaultLifecycleJobDependencies( return { now, readJobContext: readLifecycleJobContext, + readInstallRuntimeEnrichmentContext, createUnsubscribeUrl: (input, key) => createUnsubscribeActionUrl(input, key, mailRuntime().publicActionOrigin), sendRecipient: (executor, input, policy) => { diff --git a/apps/lifecycle/src/enrichment/research-input.ts b/apps/lifecycle/src/enrichment/research-input.ts index d21593920..b7dfa2cd3 100644 --- a/apps/lifecycle/src/enrichment/research-input.ts +++ b/apps/lifecycle/src/enrichment/research-input.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { isPersonalEmailDomain } from '../growth.js'; import { CompanyPageEvidenceSchema, @@ -6,22 +7,6 @@ import { type CompanyPageEvidence, } from './schema.js'; -const PERSONAL_EMAIL_DOMAINS = new Set([ - 'aol.com', - 'gmail.com', - 'googlemail.com', - 'hotmail.com', - 'icloud.com', - 'live.com', - 'me.com', - 'msn.com', - 'outlook.com', - 'proton.me', - 'protonmail.com', - 'yahoo.com', - 'ymail.com', -]); - const DomainSchema = z .string() .min(3) @@ -39,6 +24,7 @@ const FormFactsSchema = z 'contact', 'pricing', 'project-claim', + 'install_runtime', ]), emailClassification: z.enum(['work', 'personal', 'unknown']), displayName: z.string().min(1).max(120).optional(), @@ -106,7 +92,7 @@ export function buildResearchInput(candidate: unknown): ResearchInput { const researchMode = emailClassification !== 'personal' && domain && - !PERSONAL_EMAIL_DOMAINS.has(domain) + !isPersonalEmailDomain(domain) ? 'company' : 'neutral'; diff --git a/libs/growth/README.md b/libs/growth/README.md new file mode 100644 index 000000000..7ebce021f --- /dev/null +++ b/libs/growth/README.md @@ -0,0 +1,14 @@ +# Internal growth operator reports + +Run the existing CLI with the intended read-only database credentials: + +```sh +npm run growth:observability -- funnel --from 2026-09-01T00:00:00Z --to 2026-09-05T00:00:00Z +npm run growth:observability -- journey --contact 11111111-1111-4111-8111-111111111111 +``` + +`funnel` requires a positive UTC date range of at most 31 days, with an exclusive end. Independent observation/subject counts are grouped by source, kind, and install environment. Processing status reflects the current state of observations received in that window. Activation decisions use their evaluation timestamps. The linked cohort consists only of distinct contacts in those persisted decisions; its campaign outcomes include all recorded history as of the read and its authorization/stops reflect current control state (approval prerequisites, not full campaign eligibility). These counts are not a sequential conversion rate or anonymous website attribution. + +`journey` accepts one opaque contact UUID. It shows the latest 50 directly linked observations, activation decisions, jobs, and activity records, and the latest enrichment artifact's company profile and up to three source references. Each section declares its limit and truncation; missing records return `no_evidence`, an unknown contact returns `not_found`, and a deleted contact returns `redacted` with control state only. Missing profile fields mean unavailable. Profiles are candidate-domain research, not verified employment. + +Both commands are read-only and need neither the processing switch nor the email HMAC keyring. Output excludes plaintext contact/install identity, full job payloads, activity data, email drafts, and raw artifact contents. Only explicit job provenance fields are shown. Source links omit query strings and fragments; unsafe or identity-bearing links are unavailable. Persisted provider results may lag delivery; ingress rejection details require service logs. Reads are not a transaction snapshot. diff --git a/libs/growth/src/index.ts b/libs/growth/src/index.ts index 70d278c65..20ed36d38 100644 --- a/libs/growth/src/index.ts +++ b/libs/growth/src/index.ts @@ -1,4 +1,6 @@ export * from './lib/contacts.ts'; +export * from './lib/company-domain.ts'; +export { readInstallRuntimeEnrichmentContext } from './lib/observability/install-runtime-enrichment.ts'; export * from './lib/campaign-analytics.ts'; export * from './lib/crypto.ts'; export * from './lib/database.ts'; @@ -44,3 +46,7 @@ export { initializeObservationRedactions, } from './lib/observability/redaction.ts'; export type { ObservationEnrichmentReference } from './lib/observability/enrichment-contract.ts'; +export { + readGrowthFunnel, + readContactJourney, +} from './lib/observability/journey-report.ts'; diff --git a/libs/growth/src/lib/company-domain.spec.ts b/libs/growth/src/lib/company-domain.spec.ts new file mode 100644 index 000000000..63d4d0d8b --- /dev/null +++ b/libs/growth/src/lib/company-domain.spec.ts @@ -0,0 +1,52 @@ +import { + companyDomainFromEmail, + isPersonalEmailDomain, +} from './company-domain.ts'; + +describe('candidate company domains', () => { + it('normalizes only a public-looking domain without asserting employment', () => { + expect(companyDomainFromEmail('Developer+Test@Example.COM')).toBe( + 'example.com' + ); + expect(companyDomainFromEmail('Developer@sub.example.co.uk')).toBe( + 'sub.example.co.uk' + ); + }); + it.each([ + 'a@gmail.com', + 'a@PROTON.ME', + 'a@127.0.0.1', + 'a@[::1]', + 'a@localhost', + 'a@-bad.com', + 'a@bad_.com', + 'a@bad..com', + 'a@x.123', + '@example.com', + 'a@@example.com', + 'a@example.com/path', + 'a@example.com ', + ])('rejects %s', (email) => { + expect(companyDomainFromEmail(email)).toBeNull(); + }); + it('recognizes the existing personal provider set case-insensitively', () => { + for (const domain of [ + 'aol.com', + 'gmail.com', + 'googlemail.com', + 'hotmail.com', + 'icloud.com', + 'live.com', + 'me.com', + 'msn.com', + 'outlook.com', + 'proton.me', + 'protonmail.com', + 'yahoo.com', + 'ymail.com', + ]) { + expect(isPersonalEmailDomain(domain.toUpperCase())).toBe(true); + } + expect(isPersonalEmailDomain('example.com')).toBe(false); + }); +}); diff --git a/libs/growth/src/lib/company-domain.ts b/libs/growth/src/lib/company-domain.ts new file mode 100644 index 000000000..c8da34fcb --- /dev/null +++ b/libs/growth/src/lib/company-domain.ts @@ -0,0 +1,35 @@ +const PERSONAL_EMAIL_DOMAINS = new Set([ + 'aol.com', + 'gmail.com', + 'googlemail.com', + 'hotmail.com', + 'icloud.com', + 'live.com', + 'me.com', + 'msn.com', + 'outlook.com', + 'proton.me', + 'protonmail.com', + 'yahoo.com', + 'ymail.com', +]); + +export function isPersonalEmailDomain(domain: string): boolean { + return PERSONAL_EMAIL_DOMAINS.has(domain.toLowerCase()); +} + +/** An email-derived research candidate, never proof of employment or ownership. */ +export function companyDomainFromEmail(email: string): string | null { + const pieces = email.split('@'); + if (pieces.length !== 2 || !pieces[0] || /\s/u.test(email)) return null; + const domain = pieces[1].toLowerCase(); + if ( + domain.length > 253 || + !/^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$/u.test( + domain + ) || + isPersonalEmailDomain(domain) + ) + return null; + return domain; +} diff --git a/libs/growth/src/lib/jobs.spec.ts b/libs/growth/src/lib/jobs.spec.ts index f450869c7..5a01d14e2 100644 --- a/libs/growth/src/lib/jobs.spec.ts +++ b/libs/growth/src/lib/jobs.spec.ts @@ -48,8 +48,8 @@ function executorWith( ): Promise> { const marker = /\/\* growth:([a-z0-9-]+) \*\//u.exec(sql)?.[1]; if ( - sql.includes( - "pg_advisory_xact_lock_shared(hashtextextended('growth-observation-privacy-v1'" + /pg_advisory_xact_lock(?:_shared)?\(hashtextextended\('growth-observation-privacy-v1'/u.test( + sql ) ) return { rows: [] }; @@ -1679,6 +1679,7 @@ describe('job artifacts', () => { created_at: now, }; const harness = executorWith({ + 'discover-install-runtime-artifact-job': () => ({ rows: [] }), 'insert-job-artifact': (parameters, sql) => { expect(parameters).toEqual([ jobRow().id, @@ -1737,6 +1738,7 @@ describe('job artifacts', () => { created_at: now, }; const harness = executorWith({ + 'discover-install-runtime-artifact-job': () => ({ rows: [] }), 'insert-job-artifact': (parameters, sql) => { expect(parameters).toEqual([ jobRow().id, diff --git a/libs/growth/src/lib/jobs.ts b/libs/growth/src/lib/jobs.ts index f94bdc4d3..181f94a00 100644 --- a/libs/growth/src/lib/jobs.ts +++ b/libs/growth/src/lib/jobs.ts @@ -3,6 +3,7 @@ import { CONTACT_HARD_STOP_REASONS } from './contacts.ts'; import { normalizeEmail } from './crypto.ts'; import type { GrowthArtifact, GrowthJob } from './models.ts'; import { privacyLock } from './observability/store.ts'; +import { installRuntimeEvidenceSql } from './observability/install-runtime-enrichment.ts'; const FULFILLMENT_ALLOWED_PRIOR_STOPS = new Set([ 'unsubscribe', @@ -1962,6 +1963,48 @@ export async function persistJobArtifact( ]; return executor.transaction(async (transaction) => { + // Exclude ingest too: it can admit conflicting evidence after authorization. + // This short transaction takes privacy before contact/job locks and has no provider calls. + await privacyLock(transaction, true); + const installJob = await transaction.execute<{ contact_id: string | null }>( + `/* growth:discover-install-runtime-artifact-job */ + select contact_id from growth_jobs where id=$1 + and (payload->>'source'='install_runtime' or idempotency_key like 'install-runtime:v1:%')`, + [input.jobId] + ); + if (installJob.rows.length) { + if (!leaseBound || !installJob.rows[0].contact_id) + throw new JobLeaseConflictError(input.jobId); + await transaction.execute( + `/* growth:lock-install-runtime-artifact-contact */ + select id from growth_contacts where id=$1 for update`, + [installJob.rows[0].contact_id] + ); + const authorized = await transaction.execute<{ id: string }>( + `/* growth:authorize-install-runtime-artifact */ + select j.id from growth_jobs j join growth_contacts c on c.id=j.contact_id + where j.id=$1 and j.contact_id=$4 and j.kind='enrich' + and j.payload->>'source'='install_runtime' + and j.payload->>'evidence_redacted' is distinct from 'true' + and j.status='leased' and j.lease_token=$2::uuid and j.lease_until>$3 + and c.deleted_at is null and c.outreach_approved_at is not null + and not exists(select 1 from growth_activity stop where stop.contact_id=c.id + and stop.kind=any($5::text[]) and stop.occurred_at>=c.outreach_approved_at) + and ${installRuntimeEvidenceSql( + "j.payload->>'install_observation_id'", + "j.payload->>'runtime_observation_id'" + )} + for update of j`, + [ + input.jobId, + leaseToken, + now, + installJob.rows[0].contact_id, + CONTACT_HARD_STOP_REASONS, + ] + ); + if (!authorized.rows.length) throw new JobLeaseConflictError(input.jobId); + } await transaction.execute( `/* growth:insert-job-artifact */ insert into growth_artifacts ( diff --git a/libs/growth/src/lib/observability/install-runtime-enrichment.spec.ts b/libs/growth/src/lib/observability/install-runtime-enrichment.spec.ts new file mode 100644 index 000000000..a58e3c60d --- /dev/null +++ b/libs/growth/src/lib/observability/install-runtime-enrichment.spec.ts @@ -0,0 +1,52 @@ +import type { SqlExecutor } from '../database.ts'; +import { + enqueueInstallRuntimeEnrichment, + readInstallRuntimeEnrichmentContext, +} from './install-runtime-enrichment.ts'; + +describe('install/runtime enrichment boundaries', () => { + const now = new Date('2026-09-05T00:00:00Z'); + const input = { + contactId: 'c', + installObservationId: 'i', + runtimeObservationId: 'r', + email: 'developer@example.com', + now, + }; + it('does not enqueue personal-email research', async () => { + const execute = vi.fn(); + await enqueueInstallRuntimeEnrichment( + { execute }, + { ...input, email: 'a@gmail.com' } + ); + expect(execute).not.toHaveBeenCalled(); + }); + it('keeps email/domain out of the persisted job payload', async () => { + const execute = vi.fn().mockResolvedValue({ rows: [] }); + await enqueueInstallRuntimeEnrichment({ execute }, input); + const params = execute.mock.calls[0][1]; + expect(params).toContain('install-runtime:v1:c:enrich'); + expect(params).not.toContain('developer@example.com'); + expect(params).not.toContain('example.com'); + }); + it('returns only the derived domain under a short privacy transaction', async () => { + const execute = vi + .fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ + rows: [{ email_normalized: 'Developer@EXAMPLE.COM' }], + }); + const db = { + execute, + transaction: async (fn: (tx: unknown) => unknown) => fn({ execute }), + } as unknown as SqlExecutor; + expect( + await readInstallRuntimeEnrichmentContext(db, { + jobId: 'j', + leaseToken: 't', + now, + }) + ).toEqual({ companyDomain: 'example.com' }); + expect(execute.mock.calls[0][0]).toContain('pg_advisory_xact_lock_shared'); + }); +}); diff --git a/libs/growth/src/lib/observability/install-runtime-enrichment.ts b/libs/growth/src/lib/observability/install-runtime-enrichment.ts new file mode 100644 index 000000000..3a0459a7c --- /dev/null +++ b/libs/growth/src/lib/observability/install-runtime-enrichment.ts @@ -0,0 +1,93 @@ +import { CONTACT_HARD_STOP_REASONS } from '../contacts.ts'; +import { companyDomainFromEmail } from '../company-domain.ts'; +import type { SqlExecutor, SqlTransaction } from '../database.ts'; +import { privacyLock } from './store.ts'; + +/** Internal static SQL expressions only; never interpolate operator/public input. Uses contact alias c. */ +export function installRuntimeEvidenceSql( + installIdExpression: string, + runtimeIdExpression: string +): string { + return `exists ( + select 1 from growth_install_runtime_links link + join growth_observations i on i.id=link.install_observation_id + join growth_observations r on r.id=link.runtime_observation_id + join growth_observation_identities identity on identity.observation_id=i.id + where link.contact_id=c.id and link.outcome='approved' + and i.id::text=${installIdExpression} and r.id::text=${runtimeIdExpression} + and i.redacted_at is null and r.redacted_at is null + and i.source='install' and r.source='runtime' and r.kind='runtime.session_started' + and i.properties->>'environment'<>'ci' + and i.installation_token_digest=r.installation_token_digest + and i.properties->>'packageName'=r.properties->>'packageName' + and i.properties->>'packageVersion'=r.properties->>'packageVersion' + and identity.email_normalized=c.email_normalized + and not exists(select 1 from growth_observations removed + where removed.source='install' and removed.installation_token_digest=i.installation_token_digest + and removed.redacted_at is not null) + and not exists(select 1 from growth_observations other + join growth_observation_identities other_identity on other_identity.observation_id=other.id + where other.source='install' and other.installation_token_digest=i.installation_token_digest + and other_identity.email_normalized<>identity.email_normalized) + )`; +} + +/** Caller holds the shared privacy lock and has persisted the approved link in this transaction. */ +export async function enqueueInstallRuntimeEnrichment( + tx: SqlTransaction, + input: { + contactId: string; + installObservationId: string; + runtimeObservationId: string; + email: string; + now: Date; + } +): Promise { + if (!companyDomainFromEmail(input.email)) return; + await tx.execute( + `/* growth:enqueue-install-runtime-enrichment */ + insert into growth_jobs(kind,contact_id,status,available_at,idempotency_key,payload) + select 'enrich',c.id,'pending',$4,$5, + jsonb_build_object('source','install_runtime','install_observation_id',$2::text,'runtime_observation_id',$3::text) + from growth_contacts c where c.id=$1 and c.deleted_at is null and c.outreach_approved_at is not null + and not exists(select 1 from growth_activity stop where stop.contact_id=c.id + and stop.kind=any($6::text[]) and stop.occurred_at>=c.outreach_approved_at) + and ${installRuntimeEvidenceSql('$2::text', '$3::text')} + on conflict(idempotency_key) do nothing`, + [ + input.contactId, + input.installObservationId, + input.runtimeObservationId, + input.now, + `install-runtime:v1:${input.contactId}:enrich`, + CONTACT_HARD_STOP_REASONS, + ] + ); +} + +/** Re-read authorization between provider stages; returns no plaintext identity. */ +export async function readInstallRuntimeEnrichmentContext( + db: SqlExecutor, + input: { jobId: string; leaseToken: string; now: Date } +): Promise<{ companyDomain: string } | null> { + return db.transaction(async (tx) => { + await privacyLock(tx); + const result = await tx.execute<{ email_normalized: string }>( + `/* growth:read-install-runtime-enrichment-context */ + select c.email_normalized from growth_jobs j join growth_contacts c on c.id=j.contact_id + where j.id=$1 and j.kind='enrich' and j.payload->>'source'='install_runtime' + and j.status='leased' and j.lease_token=$2::uuid and j.lease_until>$3 + and c.deleted_at is null and c.outreach_approved_at is not null + and not exists(select 1 from growth_activity stop where stop.contact_id=c.id + and stop.kind=any($4::text[]) and stop.occurred_at>=c.outreach_approved_at) + and ${installRuntimeEvidenceSql( + "j.payload->>'install_observation_id'", + "j.payload->>'runtime_observation_id'" + )}`, + [input.jobId, input.leaseToken, input.now, CONTACT_HARD_STOP_REASONS] + ); + const email = result.rows[0]?.email_normalized; + const companyDomain = email ? companyDomainFromEmail(email) : null; + return companyDomain ? { companyDomain } : null; + }); +} diff --git a/libs/growth/src/lib/observability/install-runtime.ts b/libs/growth/src/lib/observability/install-runtime.ts index 543ad2261..7bb2c6e9a 100644 --- a/libs/growth/src/lib/observability/install-runtime.ts +++ b/libs/growth/src/lib/observability/install-runtime.ts @@ -2,6 +2,7 @@ import type { SqlExecutor } from '../database.ts'; import type { EmailHmacKeyring } from '../crypto.ts'; import { approveContactFromInstallRuntimeInTransaction } from '../contacts.ts'; import { privacyLock, assertIdentityKeyCoverage } from './store.ts'; +import { enqueueInstallRuntimeEnrichment } from './install-runtime-enrichment.ts'; /** Resolve admitted evidence in the existing lifecycle tick; never invoked by public payloads. */ export async function processInstallRuntimeActivations( @@ -103,6 +104,15 @@ export async function processInstallRuntimeActivations( values($1,$2,$3,$4,$5)`, [runtime.id, installation?.id ?? null, contactId, outcome, input.now] ); + if (outcome === 'approved' && contactId && installation?.email_normalized) { + await enqueueInstallRuntimeEnrichment(tx, { + contactId, + installObservationId: installation.id, + runtimeObservationId: runtime.id, + email: installation.email_normalized, + now: input.now, + }); + } counts[outcome]++; } return counts; diff --git a/libs/growth/src/lib/observability/journey-report.spec.ts b/libs/growth/src/lib/observability/journey-report.spec.ts new file mode 100644 index 000000000..c024753e2 --- /dev/null +++ b/libs/growth/src/lib/observability/journey-report.spec.ts @@ -0,0 +1,91 @@ +import { readContactJourney, readGrowthFunnel } from './journey-report.ts'; +import type { SqlExecutor } from '../database.ts'; + +function executor(rows: Record[][] = []) { + const execute = vi.fn(async () => ({ rows: rows.shift() ?? [] })); + return { execute, transaction: vi.fn() } as unknown as SqlExecutor; +} +describe('bounded journey reports', () => { + it.each([ + [ + 'https://example.invalid/about?email=person%40example.org#private', + 'https://example.invalid/about', + ], + ['https://example.invalid/person%40example.org', null], + ['https://example.invalid/person%2540example.org', null], + ['https://example.invalid/person@example.org', null], + ['https://user:password@example.invalid/about', null], + ['http://example.invalid/about', null], + ['javascript:alert(1)', null], + ['not a URL', null], + ])( + 'removes identity and unsafe URL context from %s', + async (url, expected) => { + const id = '11111111-1111-4111-8111-111111111111'; + const db = executor([ + [{ id }], + [ + { + id, + updated_at: '2026-01-01', + deleted_at: null, + outreach_approved_at: null, + latest_hard_stop_kind: null, + latest_hard_stop_at: null, + }, + ], + [], + [], + [], + [], + [{ sources: [{ id: 'source1', url }] }], + ]); + const result = await readContactJourney(db, id); + expect(result.enrichment?.latest[0]).toMatchObject({ + sources: [{ id: 'source1', url: expected }], + }); + } + ); + it('validates range and identity before querying', async () => { + const db = executor(); + await expect( + readGrowthFunnel(db, { + from: new Date('2026-01-01'), + to: new Date('2026-03-01'), + }) + ).rejects.toThrow(); + await expect( + readContactJourney(db, 'email@example.invalid') + ).rejects.toThrow(); + expect(db.execute).not.toHaveBeenCalled(); + }); + it('distinguishes missing contacts from empty evidence', async () => { + expect( + await readContactJourney( + executor(), + '11111111-1111-4111-8111-111111111111' + ) + ).toMatchObject({ state: 'not_found' }); + }); + it('returns only control state for deleted contacts', async () => { + const db = executor([ + [{ id: '11111111-1111-4111-8111-111111111111' }], + [ + { + id: '11111111-1111-4111-8111-111111111111', + deleted_at: '2026-01-01', + updated_at: '2026-01-01', + outreach_approved_at: null, + latest_hard_stop_kind: null, + latest_hard_stop_at: null, + }, + ], + ]); + const result = await readContactJourney( + db, + '11111111-1111-4111-8111-111111111111' + ); + expect(result).toMatchObject({ state: 'redacted' }); + expect(db.execute).toHaveBeenCalledTimes(2); + }); +}); diff --git a/libs/growth/src/lib/observability/journey-report.ts b/libs/growth/src/lib/observability/journey-report.ts new file mode 100644 index 000000000..096478a22 --- /dev/null +++ b/libs/growth/src/lib/observability/journey-report.ts @@ -0,0 +1,204 @@ +import type { SqlExecutor } from '../database.ts'; +import { + CONTACT_HARD_STOP_REASONS, + readContactControlState, +} from '../contacts.ts'; +import { ObservationError, uuid } from './contracts.ts'; + +const LIMIT = 50; +function reportSourceUrl(value: unknown): string | null { + if (typeof value !== 'string' || value.length > 500) return null; + try { + const url = new URL(value); + // Encoded paths can hide identity through repeated decoding. Omit them + // conservatively; query and fragment are never useful source provenance. + if ( + url.protocol !== 'https:' || + url.username || + url.password || + /[%@]/u.test(url.pathname) + ) + return null; + url.search = ''; + url.hash = ''; + return url.toString(); + } catch { + return null; + } +} + +function bounded(rows: Record[], limit = LIMIT) { + return { + state: rows.length ? 'available' : 'no_evidence', + latest: rows.slice(0, limit), + limit, + truncated: rows.length > limit, + }; +} + +/** Independent signals are not a conversion denominator. Cohort membership requires a persisted link. */ +export async function readGrowthFunnel( + db: SqlExecutor, + input: { from: Date; to: Date } +) { + if ( + !Number.isFinite(input.from.getTime()) || + !Number.isFinite(input.to.getTime()) || + input.to <= input.from || + input.to.getTime() - input.from.getTime() > 31 * 86400000 + ) + throw new ObservationError('invalid_payload'); + const parameters = [input.from, input.to]; + const signals = await db.execute( + `select source,kind,case when source='install' then properties->>'environment' end as environment, + count(*) as observations,count(distinct subject_id) as subjects + from growth_observations where received_at >= $1 and received_at < $2 + group by source,kind,case when source='install' then properties->>'environment' end order by source,kind,environment`, + parameters + ); + const links = await db.execute( + `select outcome,count(*) as runtime_observations,count(distinct contact_id) as contacts + from growth_install_runtime_links where evaluated_at >= $1 and evaluated_at < $2 group by outcome order by outcome`, + parameters + ); + const processing = await db.execute( + `select coalesce(w.status,'missing') as status,count(*) as observations + from growth_observations o left join growth_observation_work w on w.observation_id=o.id + where o.received_at >= $1 and o.received_at < $2 group by w.status order by status`, + parameters + ); + const cohort = await db.execute( + `with cohort as ( + select distinct contact_id from growth_install_runtime_links + where evaluated_at >= $1 and evaluated_at < $2 and contact_id is not null + ), states as ( + select c.id,c.deleted_at,c.outreach_approved_at,s.kind as stop_kind,s.occurred_at as stop_at + from cohort join growth_contacts c on c.id=cohort.contact_id + left join lateral (select kind,occurred_at from growth_activity where contact_id=c.id and kind=any($3::text[]) + order by occurred_at desc,id desc limit 1) s on true + ) select count(*) as linked_contacts, + count(*) filter(where deleted_at is null and outreach_approved_at is not null and (stop_at is null or stop_at < outreach_approved_at) and stop_kind is distinct from 'deletion') as currently_authorized_contacts, + count(*) filter(where exists(select 1 from growth_activity a where a.contact_id=states.id and a.kind='campaign.enrolled:v1')) as enrolled_contacts, + count(*) filter(where exists(select 1 from growth_jobs j where j.contact_id=states.id and j.kind='send_step' and j.delivery_status <> 'not_submitted')) as submitted_or_attempted_contacts, + count(*) filter(where exists(select 1 from growth_activity a where a.contact_id=states.id and a.kind='campaign.step_accepted')) as accepted_contacts, + count(*) filter(where exists(select 1 from growth_activity a where a.contact_id=states.id and a.kind='delivery.delivered')) as delivered_contacts, + count(*) filter(where exists(select 1 from growth_activity a where a.contact_id=states.id and a.kind='campaign.reply_received')) as replied_contacts, + count(*) filter(where deleted_at is not null or stop_kind='deletion' or (stop_at is not null and (outreach_approved_at is null or stop_at >= outreach_approved_at))) as currently_stopped_contacts, + count(*) filter(where exists(select 1 from growth_artifacts a join growth_jobs j on j.id=a.job_id where a.contact_id=states.id and j.kind='enrich' and a.kind='enrichment.v1' and a.schema_version=1)) as enriched_contacts + from states`, + [...parameters, CONTACT_HARD_STOP_REASONS] + ); + return { + from: input.from.toISOString(), + to: input.to.toISOString(), + independentSignals: { + unit: 'observations and distinct subjects per group; subjects may appear in multiple groups', + window: 'received_at [from,to)', + rows: signals.rows, + }, + activationDecisions: { + unit: 'runtime observations and distinct contacts per outcome', + window: 'evaluated_at [from,to)', + rows: links.rows, + }, + observationProcessing: { + unit: 'observations received in the window, current processing state', + rows: processing.rows, + }, + linkedContactCohort: { + definition: + 'Distinct non-null contacts linked by decisions evaluated in the window. Outcome counts are all persisted history as of this read; current authorization/stops use current control state, not full campaign eligibility. Stages are not necessarily sequential.', + unit: 'distinct contacts', + counts: cohort.rows[0] ?? null, + }, + unavailable: [ + 'Anonymous website-to-install attribution is not persisted.', + 'Ingress rejection details require service logs.', + 'Provider outcomes not yet persisted are unavailable.', + ], + }; +} + +/** Read a bounded, identity-free operator view. Never select job payloads, activity data or raw artifacts. */ +export async function readContactJourney(db: SqlExecutor, contactId: string) { + uuid(contactId); + const contact = await db.execute( + 'select id from growth_contacts where id=$1', + [contactId] + ); + if (!contact.rows[0]) return { contactId, state: 'not_found' }; + const control = await readContactControlState(db, contactId); + if (control.authorization === 'deleted') + return { contactId, state: 'redacted', control }; + const observations = await db.execute( + `select o.id,o.subject_id,o.source,o.kind,o.occurred_at,o.received_at,o.trust, + o.redacted_at is not null as redacted,w.status as processing_status,w.last_error_code + from growth_observations o left join growth_observation_work w on w.observation_id=o.id + where exists(select 1 from growth_install_runtime_links l where l.contact_id=$1 and (l.runtime_observation_id=o.id or l.install_observation_id=o.id)) + or exists(select 1 from growth_observation_form_links f where f.contact_id=$1 and f.observation_id=o.id) + order by o.received_at desc,o.id desc limit $2`, + [contactId, LIMIT + 1] + ); + const activation = await db.execute( + `select runtime_observation_id,install_observation_id,outcome,evaluated_at + from growth_install_runtime_links where contact_id=$1 order by evaluated_at desc,runtime_observation_id desc limit $2`, + [contactId, LIMIT + 1] + ); + const jobs = await db.execute( + `select id,kind,status,delivery_status,attempts,available_at,created_at,updated_at,last_error_code, + payload->>'campaign_version' as campaign_version,payload->>'step' as step, + payload->>'source' as source,payload->>'install_observation_id' as install_observation_id,payload->>'runtime_observation_id' as runtime_observation_id + from growth_jobs where contact_id=$1 order by created_at desc,id desc limit $2`, + [contactId, LIMIT + 1] + ); + const activity = await db.execute( + `select id,kind,occurred_at from growth_activity where contact_id=$1 + order by occurred_at desc,id desc limit $2`, + [contactId, LIMIT + 1] + ); + const artifacts = await db.execute( + `select a.id,a.job_id,a.kind,a.schema_version,a.created_at, + left(a.content->'company_profile'->>'name',120) as company_name, + left(a.content->'company_profile'->>'description',500) as company_description, + left(a.content->'company_profile'->>'industry',120) as company_industry, + (select jsonb_agg(jsonb_build_object('id',left(s->>'id',40),'url',left(s->>'url',500),'retrieved_at',left(s->>'retrieved_at',40),'content_hash',left(s->>'content_hash',64))) + from (select s from jsonb_array_elements(case when jsonb_typeof(a.content->'sources')='array' then a.content->'sources' else '[]'::jsonb end) s limit 3) sources) as sources, + case when jsonb_typeof(a.content->'sources')='array' then jsonb_array_length(a.content->'sources')>3 else false end as sources_truncated + from growth_artifacts a join growth_jobs j on j.id=a.job_id + where a.contact_id=$1 and j.kind='enrich' and a.kind='enrichment.v1' and a.schema_version=1 + and not exists(select 1 from growth_contacts c where c.id=$1 and c.deleted_at is not null) + order by a.created_at desc,a.id desc limit 2`, + [contactId] + ); + // Company prose is untrusted; avoid accidentally echoing an email embedded in a profile or URL. + const safeArtifacts = JSON.parse( + JSON.stringify( + artifacts.rows.map((artifact) => ({ + ...artifact, + sources: Array.isArray(artifact.sources) + ? artifact.sources.map((source: Record) => ({ + ...source, + url: reportSourceUrl(source.url), + })) + : artifact.sources, + })) + ).replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/giu, '[redacted]') + ) as Record[]; + return { + contactId, + state: 'available', + control, + observations: bounded(observations.rows), + activation: bounded(activation.rows), + jobs: bounded(jobs.rows), + activity: bounded(activity.rows), + enrichment: bounded(safeArtifacts, 1), + notes: [ + 'Latest evidence only; truncation is explicit per section.', + 'Only persisted direct observation links are shown; anonymous browsing is unavailable.', + 'Company research is a candidate-domain profile, not verified employment. Missing profile fields mean unavailable.', + 'Only enrichment.v1 schema 1 artifacts are summarized. Source URLs omit query/fragment; unsafe or encoded paths are unavailable.', + 'Control state is current; earlier activation approval does not override stops. Reads are not a transaction snapshot.', + ], + }; +} diff --git a/libs/growth/src/lib/observability/redaction.ts b/libs/growth/src/lib/observability/redaction.ts index 7e3e82db7..3ad69a0f0 100644 --- a/libs/growth/src/lib/observability/redaction.ts +++ b/libs/growth/src/lib/observability/redaction.ts @@ -60,6 +60,36 @@ async function redactLocked( [key.digest, key.keyVersion, now] ); const ids = rows.rows.map((r) => r.observation_id); + // Retire derived research before removing its identity evidence. A redacted + // install invalidates every version sharing its token, just as authorization does. + await tx.execute( + `/* growth:redact-install-runtime-enrichment */ + with affected as materialized ( + select j.id from growth_jobs j + where j.kind='enrich' and j.payload->>'source'='install_runtime' + and ( + j.payload->>'install_observation_id'=any($1::text[]) + or j.payload->>'runtime_observation_id'=any($1::text[]) + or exists ( + select 1 from growth_observations linked_install + join growth_observations removed on removed.installation_token_digest=linked_install.installation_token_digest + where linked_install.id::text=j.payload->>'install_observation_id' + and removed.id=any($1::uuid[]) and removed.source='install' + ) + ) + order by j.id for update of j + ), scrubbed as ( + update growth_jobs j set + status=case when j.status in ('pending','leased') then 'cancelled' else j.status end, + lease_token=null, lease_until=null, + payload=jsonb_build_object('source','install_runtime','evidence_redacted',true), + last_error_code='install_runtime_evidence_redacted', updated_at=$2 + where j.id in (select id from affected) + returning j.id + ) + delete from growth_artifacts where job_id in (select id from scrubbed)`, + [ids, now] + ); await tx.execute( 'select observation_id from growth_observation_work where observation_id=any($1::uuid[]) order by observation_id for update', [ids] diff --git a/libs/growth/test/install-runtime-enrichment.integration.spec.ts b/libs/growth/test/install-runtime-enrichment.integration.spec.ts new file mode 100644 index 000000000..bb00dbb14 --- /dev/null +++ b/libs/growth/test/install-runtime-enrichment.integration.spec.ts @@ -0,0 +1,451 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { + enqueueInstallRuntimeEnrichment, + readInstallRuntimeEnrichmentContext, +} from '../src/lib/observability/install-runtime-enrichment.ts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { privacyLock } from '../src/lib/observability/store.ts'; +import { persistJobArtifact } from '../src/lib/jobs.ts'; +import { redactObservationEvidence } from '../src/lib/observability/redaction.ts'; +import { stopContact } from '../src/lib/stops.ts'; +import { createEmailLookupHmac } from '../src/lib/crypto.ts'; +import { + cleanEvidence, + evidenceDatabase, + evidenceFixture, + evidenceKeys, +} from './observability-fixtures.ts'; + +describe('install/runtime enrichment SQL authorization', () => { + let db: SqlExecutor; + let contactId: string, leaseToken: string, token: string; + let subjects: string[], operations: string[]; + const now = new Date(); + let installObservationId: string, + runtimeObservationId: string, + email: string, + jobId: string; + beforeEach(async () => { + contactId = randomUUID(); + leaseToken = randomUUID(); + token = randomUUID(); + subjects = []; + operations = []; + db = await evidenceDatabase(); + const install = evidenceFixture(now); + const fixtureEmail = install.events[0].identity?.gitEmail; + if (!fixtureEmail) throw new Error('fixture_email_required'); + email = fixtureEmail; + install.events[0].installationToken = token; + install.events[0].properties.environment = 'unknown'; + install.events[0].properties.environmentEvidence = 'unknown'; + const runtimeSubject = randomUUID(); + const runtimeEvent = randomUUID(); + subjects.push(install.events[0].subject.id, runtimeSubject); + await acceptObservationBatch(db, 'install', install, { + now, + keyring: evidenceKeys, + }); + await acceptObservationBatch( + db, + 'runtime', + { + schemaVersion: 1, + events: [ + { + eventId: runtimeEvent, + sessionId: randomUUID(), + kind: 'runtime.session_started', + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: runtimeSubject, + namespace: 'development_browser', + scope: 'memory', + }, + installationToken: token, + properties: { + packageName: '@threadplane/chat', + packageVersion: '1', + integration: 'langgraph', + }, + }, + ], + }, + { now } + ); + installObservationId = ( + await db.execute<{ id: string }>( + 'select id from growth_observations where event_id=$1', + [install.events[0].eventId] + ) + ).rows[0].id; + runtimeObservationId = ( + await db.execute<{ id: string }>( + 'select id from growth_observations where event_id=$1', + [runtimeEvent] + ) + ).rows[0].id; + await db.execute( + `insert into growth_contacts(id,email_normalized,email_lookup_hmac,email_hmac_key_version,source,outreach_approved_at) values($1,$2,$3,777,'install_runtime',$4)`, + [contactId, email, randomUUID(), now] + ); + await db.execute( + `insert into growth_install_runtime_links(runtime_observation_id,install_observation_id,contact_id,outcome,evaluated_at) values($1,$2,$3,'approved',$4)`, + [runtimeObservationId, installObservationId, contactId, now] + ); + await enqueue(); + jobId = ( + await db.execute<{ id: string }>( + 'select id from growth_jobs where contact_id=$1', + [contactId] + ) + ).rows[0].id; + await db.execute( + `update growth_jobs set status='leased',lease_token=$2,lease_until=$3 where id=$1`, + [jobId, leaseToken, new Date(now.getTime() + 60_000)] + ); + }); + afterEach(async () => { + if (!db) return; + await db.execute( + "delete from growth_observation_redactions where selector_kind='email' and selector_key=$1 and key_version=$2", + [ + createEmailLookupHmac(email, evidenceKeys.active).digest, + evidenceKeys.active.version, + ] + ); + await db.execute('delete from growth_artifacts where contact_id=$1', [ + contactId, + ]); + await cleanEvidence(db, subjects, operations); + await db.execute('delete from growth_jobs where contact_id=$1', [ + contactId, + ]); + await db.execute('delete from growth_activity where contact_id=$1', [ + contactId, + ]); + await db.execute('delete from growth_contacts where id=$1', [contactId]); + await db.close?.(); + }); + async function enqueue() { + await db.transaction(async (tx) => { + await privacyLock(tx); + await enqueueInstallRuntimeEnrichment(tx, { + contactId, + installObservationId, + runtimeObservationId, + email, + now, + }); + }); + } + async function context() { + return readInstallRuntimeEnrichmentContext(db, { jobId, leaseToken, now }); + } + it('serializes admitted evidence writes through the final artifact insertion', async () => { + let authorized!: () => void; + let resume!: () => void; + const atAuthorization = new Promise((resolve) => { + authorized = resolve; + }); + const resumed = new Promise((resolve) => { + resume = resolve; + }); + const pausedDb: SqlExecutor = { + execute: db.execute, + transaction: (operation) => + db.transaction((tx) => + operation({ + async execute>( + sql: string, + parameters?: readonly unknown[] + ) { + const result = await tx.execute(sql, parameters); + if (sql.includes('growth:authorize-install-runtime-artifact')) { + authorized(); + await resumed; + } + return result; + }, + }) + ), + }; + const writing = persistJobArtifact(pausedDb, { + jobId, + leaseToken, + now, + kind: 'enrichment.v1', + schemaVersion: 1, + content: { summary: 'Concurrent persistence' }, + }); + const ingestionCanLock = () => + db.transaction( + async (tx) => + ( + await tx.execute<{ acquired: boolean }>( + "select pg_try_advisory_xact_lock_shared(hashtextextended('growth-observation-privacy-v1',0)) as acquired" + ) + ).rows[0].acquired + ); + try { + await Promise.race([ + atAuthorization, + writing.then(() => { + throw new Error('authorization_pause_not_reached'); + }), + ]); + // Ingest uses this shared lock before admitting conflicting identities. + expect(await ingestionCanLock()).toBe(false); + } finally { + resume(); + await writing; + } + expect(await ingestionCanLock()).toBe(true); + }); + async function persist() { + return persistJobArtifact(db, { + jobId, + leaseToken, + now, + kind: 'enrichment.v1', + schemaVersion: 1, + content: { summary: 'Synthetic company research' }, + }); + } + async function redactRuntime() { + const subjectId = ( + await db.execute<{ subject_id: string }>( + 'select subject_id from growth_observations where id=$1', + [runtimeObservationId] + ) + ).rows[0].subject_id; + const operationId = randomUUID(); + operations.push(operationId); + await redactObservationEvidence( + db, + { subjectId }, + { operationId, now, keyring: evidenceKeys } + ); + } + it('rejects artifact persistence after a linked observation is redacted', async () => { + await redactRuntime(); + await expect(persist()).rejects.toThrow(); + await expect( + persistJobArtifact(db, { + jobId, + kind: 'enrichment.v1', + schemaVersion: 1, + content: { summary: 'Late unbound writer' }, + }) + ).rejects.toThrow(); + expect( + ( + await db.execute( + 'select id from growth_artifacts where contact_id=$1', + [contactId] + ) + ).rows + ).toHaveLength(0); + }); + it('rejects artifact persistence by a worker leased before an authoritative stop', async () => { + await stopContact(db, { + contactId, + reason: 'unsubscribe', + eventKey: randomUUID(), + occurredAt: now, + source: 'integration-test', + provenance: { kind: 'one_click', policyVersion: 'test:v1' }, + }); + await expect(persist()).rejects.toThrow(); + expect( + ( + await db.execute( + 'select id from growth_artifacts where contact_id=$1', + [contactId] + ) + ).rows + ).toHaveLength(0); + }); + it('redaction removes completed research, scrubs provenance payloads and cancels a leased worker', async () => { + await persist(); + await db.execute( + `update growth_jobs set status='completed',lease_token=null,lease_until=null where id=$1`, + [jobId] + ); + const leasedJobId = randomUUID(); + await db.execute( + `insert into growth_jobs(id,kind,contact_id,status,available_at,idempotency_key,payload,lease_token,lease_until) + select $2::uuid,kind,contact_id,'leased',available_at,$2::text,payload,$3::uuid,$4::timestamptz from growth_jobs where id=$1`, + [jobId, leasedJobId, leaseToken, new Date(now.getTime() + 60_000)] + ); + await redactRuntime(); + expect( + ( + await db.execute( + 'select id from growth_artifacts where contact_id=$1', + [contactId] + ) + ).rows + ).toHaveLength(0); + const jobs = ( + await db.execute<{ + id: string; + status: string; + payload: unknown; + lease_token: string | null; + }>( + 'select id,status,payload,lease_token from growth_jobs where contact_id=$1', + [contactId] + ) + ).rows; + expect(jobs.find((job) => job.id === jobId)).toMatchObject({ + status: 'completed', + payload: { source: 'install_runtime', evidence_redacted: true }, + }); + expect(jobs.find((job) => job.id === leasedJobId)).toMatchObject({ + status: 'cancelled', + payload: { source: 'install_runtime', evidence_redacted: true }, + lease_token: null, + }); + }); + it('redacting another package version on the same token scrubs completed research', async () => { + await persist(); + await db.execute( + "update growth_jobs set status='completed',lease_token=null,lease_until=null where id=$1", + [jobId] + ); + const other = evidenceFixture(now); + other.events[0].installationToken = token; + other.events[0].properties.packageVersion = '2'; + other.events[0].identity = { gitEmail: email }; + subjects.push(other.events[0].subject.id); + await acceptObservationBatch(db, 'install', other, { + now, + keyring: evidenceKeys, + }); + const subjectId = ( + await db.execute<{ subject_id: string }>( + 'select subject_id from growth_observations where event_id=$1', + [other.events[0].eventId] + ) + ).rows[0].subject_id; + const operationId = randomUUID(); + operations.push(operationId); + await redactObservationEvidence( + db, + { subjectId }, + { operationId, now, keyring: evidenceKeys } + ); + expect( + ( + await db.execute( + 'select id from growth_artifacts where contact_id=$1', + [contactId] + ) + ).rows + ).toHaveLength(0); + expect( + ( + await db.execute<{ payload: unknown }>( + 'select payload from growth_jobs where id=$1', + [jobId] + ) + ).rows[0].payload + ).toEqual({ source: 'install_runtime', evidence_redacted: true }); + }); + it('enqueues once with no form/enrollment and blocks stale leases, stops, CI and redacted evidence', async () => { + await enqueue(); + await enqueue(); + const jobs = ( + await db.execute<{ id: string; payload: unknown }>( + 'select id,payload from growth_jobs where contact_id=$1', + [contactId] + ) + ).rows; + expect(jobs).toHaveLength(1); + jobId = jobs[0].id; + expect(jobs[0].payload).toEqual({ + source: 'install_runtime', + install_observation_id: installObservationId, + runtime_observation_id: runtimeObservationId, + }); + await db.execute( + `update growth_jobs set status='leased',lease_token=$2,lease_until=$3 where id=$1`, + [jobId, leaseToken, new Date(now.getTime() + 60_000)] + ); + expect(await context()).toEqual({ companyDomain: 'example.invalid' }); + expect( + await readInstallRuntimeEnrichmentContext(db, { + jobId, + leaseToken: randomUUID(), + now, + }) + ).toBeNull(); + expect( + await readInstallRuntimeEnrichmentContext(db, { + jobId, + leaseToken, + now: new Date(now.getTime() + 60_000), + }) + ).toBeNull(); + await db.execute( + `insert into growth_activity(event_key,contact_id,kind,occurred_at) values($1,$2,'unsubscribe',$3)`, + [randomUUID(), contactId, now] + ); + expect(await context()).toBeNull(); + await db.execute('delete from growth_activity where contact_id=$1', [ + contactId, + ]); + await db.execute( + `update growth_observations set properties=jsonb_set(properties,'{environment}','"ci"') where id=$1`, + [installObservationId] + ); + expect(await context()).toBeNull(); + await db.execute( + `update growth_observations set properties=jsonb_set(properties,'{environment}','"unknown"') where id=$1`, + [installObservationId] + ); + await db.execute( + 'update growth_observations set redacted_at=$2 where id=$1', + [runtimeObservationId, now] + ); + expect(await context()).toBeNull(); + await db.execute( + 'update growth_observations set redacted_at=null where id=$1', + [runtimeObservationId] + ); + expect(await context()).not.toBeNull(); + await db.execute('update growth_contacts set deleted_at=$2 where id=$1', [ + contactId, + now, + ]); + expect(await context()).toBeNull(); + await db.execute('update growth_contacts set deleted_at=null where id=$1', [ + contactId, + ]); + }); + it('rejects a conflicting email on the same token even across package versions', async () => { + const conflict = evidenceFixture(now); + conflict.events[0].installationToken = token; + conflict.events[0].properties.packageVersion = '2'; + subjects.push(conflict.events[0].subject.id); + await acceptObservationBatch(db, 'install', conflict, { + now, + keyring: evidenceKeys, + }); + expect(await context()).toBeNull(); + await db.execute('delete from growth_jobs where contact_id=$1', [ + contactId, + ]); + await enqueue(); + expect( + ( + await db.execute('select id from growth_jobs where contact_id=$1', [ + contactId, + ]) + ).rows + ).toHaveLength(0); + }); +}); diff --git a/libs/growth/test/install-runtime.integration.spec.ts b/libs/growth/test/install-runtime.integration.spec.ts index 75041c07b..499c08b99 100644 --- a/libs/growth/test/install-runtime.integration.spec.ts +++ b/libs/growth/test/install-runtime.integration.spec.ts @@ -103,7 +103,7 @@ describe('install-runtime founder activation', () => { emails.push(install.events[0].identity!.gitEmail!); return { install, runtime }; } - it('resolves a runtime that arrived before install and enrolls once without enrichment', async () => { + it('resolves a runtime that arrived before install and enrolls without waiting for optional enrichment', async () => { const now = new Date(); const { install, runtime } = fixture(now); await acceptObservationBatch(db, 'runtime', runtime, { @@ -158,10 +158,14 @@ describe('install-runtime founder activation', () => { batchSize: 20, }); const jobs = await db.execute<{ id: string }>( - "select id from growth_jobs where contact_id=$1 order by payload->>'step'", + "select id from growth_jobs where contact_id=$1 and kind='send_step' order by payload->>'step'", [contact.id] ); expect(jobs.rows).toHaveLength(3); + const enrichment = await db.execute<{status: string; payload: Record}>( + "select status,payload from growth_jobs where contact_id=$1 and kind='enrich'", [contact.id] + ); + expect(enrichment.rows).toEqual([expect.objectContaining({status: 'pending', payload: expect.objectContaining({source: 'install_runtime'})})]); expect( await readLifecycleJobContext(db, { jobId: jobs.rows[0].id }) ).toMatchObject({ campaignEnrollmentReason: 'install_runtime' }); diff --git a/libs/growth/test/journey-report.integration.spec.ts b/libs/growth/test/journey-report.integration.spec.ts new file mode 100644 index 000000000..6d8858e8d --- /dev/null +++ b/libs/growth/test/journey-report.integration.spec.ts @@ -0,0 +1,219 @@ +import { randomUUID } from 'node:crypto'; +import type { SqlExecutor } from '../src/lib/database.ts'; +import { + readContactJourney, + readGrowthFunnel, +} from '../src/lib/observability/journey-report.ts'; +import { acceptObservationBatch } from '../src/lib/observability/ingest.ts'; +import { + cleanEvidence, + evidenceDatabase, + evidenceFixture, + evidenceKeys, +} from './observability-fixtures.ts'; + +describe('operator journey SQL joins and privacy', () => { + let db: SqlExecutor; + const contactId = randomUUID(), + otherContact = randomUUID(), + jobId = randomUUID(); + const subjects: string[] = []; + const now = new Date('2026-08-02T12:00:00Z'); + beforeAll(async () => { + db = await evidenceDatabase(); + }); + afterAll(async () => { + await db.execute( + 'delete from growth_artifacts where contact_id=any($1::uuid[])', + [[contactId, otherContact]] + ); + await cleanEvidence(db, subjects); + await db.execute( + 'delete from growth_jobs where contact_id=any($1::uuid[])', + [[contactId, otherContact]] + ); + await db.execute( + 'delete from growth_activity where contact_id=any($1::uuid[])', + [[contactId, otherContact]] + ); + await db.execute('delete from growth_contacts where id=any($1::uuid[])', [ + [contactId, otherContact], + ]); + await db.close?.(); + }); + it('deduplicates linked contacts, excludes unrelated outcomes, bounds evidence and redacts deleted profiles', async () => { + for (const id of [contactId, otherContact]) { + await db.execute( + `insert into growth_contacts(id,email_normalized,email_lookup_hmac,email_hmac_key_version,source,outreach_approved_at) + values($1,$2,$3,777,'test',$4)`, + [id, `${id}@private.invalid`, randomUUID(), now] + ); + } + const batch = evidenceFixture(now); + subjects.push(batch.events[0].subject.id); + await acceptObservationBatch(db, 'install', batch, { + now, + keyring: evidenceKeys, + }); + const installId = ( + await db.execute<{ id: string }>( + 'select id from growth_observations where event_id=$1', + [batch.events[0].eventId] + ) + ).rows[0].id; + // Two separately collected runtime observations link to the same admitted installation/contact. + const runtimeSubject = randomUUID(); + subjects.push(runtimeSubject); + const events = [1, 2].map(() => ({ + eventId: randomUUID(), + sessionId: randomUUID(), + kind: 'runtime.session_started', + occurredAt: now.toISOString(), + collectorVersion: '1', + subject: { + id: runtimeSubject, + namespace: 'development_browser', + scope: 'memory', + }, + properties: { + packageName: '@threadplane/langgraph', + packageVersion: '1', + integration: 'langgraph', + }, + })); + await acceptObservationBatch( + db, + 'runtime', + { schemaVersion: 1, events }, + { now } + ); + for (const event of events) + await db.execute( + `insert into growth_install_runtime_links(runtime_observation_id,install_observation_id,contact_id,outcome,evaluated_at) + select id,$2,$3,'approved',$4 from growth_observations where event_id=$1`, + [event.eventId, installId, contactId, now] + ); + for (const [id, kind] of [ + [contactId, 'campaign.enrolled:v1'], + [otherContact, 'campaign.reply_received'], + ]) { + await db.execute( + 'insert into growth_activity(event_key,contact_id,kind,occurred_at,data) values($1,$2,$3,$4,$5::jsonb)', + [ + randomUUID(), + id, + kind, + now, + JSON.stringify({ email: 'hidden@private.invalid' }), + ] + ); + } + await db.execute( + `insert into growth_jobs(id,kind,contact_id,status,available_at,idempotency_key,payload) values($1,'enrich',$2,'completed',$3,$4,$5::jsonb)`, + [ + jobId, + contactId, + now, + randomUUID(), + JSON.stringify({ + source: 'install_runtime', + install_observation_id: installId, + email: 'hidden@private.invalid', + }), + ] + ); + await db.execute( + `insert into growth_artifacts(job_id,contact_id,kind,schema_version,content) values($1,$2,'enrichment.v1',1,$3::jsonb)`, + [ + jobId, + contactId, + JSON.stringify({ + company_profile: { + name: 'Example', + description: 'A company', + industry: 'Software', + }, + sources: [ + { + id: 'source1', + url: 'https://example.invalid/about', + retrieved_at: now.toISOString(), + content_hash: 'a'.repeat(64), + }, + ], + summary: 'hidden@private.invalid', + drafts: ['private content'], + }), + ] + ); + for (const [kind, schemaVersion] of [ + ['unrelated', 1], + ['enrichment.v1', 2], + ] as const) { + await db.execute( + 'update growth_artifacts set kind=$2,schema_version=$3 where job_id=$1', + [jobId, kind, schemaVersion] + ); + const excluded = await readGrowthFunnel(db, { + from: new Date('2026-08-02'), + to: new Date('2026-08-03'), + }); + expect(excluded.linkedContactCohort.counts).toMatchObject({ + enriched_contacts: '0', + }); + expect((await readContactJourney(db, contactId)).enrichment?.state).toBe( + 'no_evidence' + ); + } + await db.execute( + "update growth_artifacts set kind='enrichment.v1',schema_version=1 where job_id=$1", + [jobId] + ); + const funnel = await readGrowthFunnel(db, { + from: new Date('2026-08-02'), + to: new Date('2026-08-03'), + }); + expect(funnel.linkedContactCohort.counts).toMatchObject({ + linked_contacts: '1', + enrolled_contacts: '1', + replied_contacts: '0', + enriched_contacts: '1', + }); + expect(funnel.activationDecisions.rows).toContainEqual({ + outcome: 'approved', + runtime_observations: '2', + contacts: '1', + }); + const journey = await readContactJourney(db, contactId); + expect(journey.observations?.latest).toHaveLength(3); + expect(journey.enrichment?.latest[0]).toMatchObject({ + company_name: 'Example', + sources: [ + { + id: 'source1', + url: 'https://example.invalid/about', + retrieved_at: now.toISOString(), + content_hash: 'a'.repeat(64), + }, + ], + }); + expect(JSON.stringify(journey)).not.toMatch( + /private.invalid|private content|Synthetic Developer/ + ); + await db.execute( + `insert into growth_activity(event_key,contact_id,kind,occurred_at) + select $1 || n,$2,'test.event',$3 from generate_series(1,55) n`, + [randomUUID(), contactId, now] + ); + const truncated = await readContactJourney(db, contactId); + expect(truncated.activity?.latest).toHaveLength(50); + expect(truncated.activity?.truncated).toBe(true); + await db.execute('update growth_contacts set deleted_at=$2 where id=$1', [ + contactId, + now, + ]); + const deleted = await readContactJourney(db, contactId); + expect(deleted.state).toBe('redacted'); + expect(JSON.stringify(deleted)).not.toContain('Example'); + }); +}); diff --git a/scripts/growth-observability.mts b/scripts/growth-observability.mts index 0f07ebd68..5d08a5555 100644 --- a/scripts/growth-observability.mts +++ b/scripts/growth-observability.mts @@ -14,6 +14,8 @@ import { type SqlExecutor, type EmailHmacKeyring, collectionSource, + readGrowthFunnel, + readContactJourney, } from '../libs/growth/src/index.ts'; import { uuid } from '../libs/growth/src/lib/observability/contracts.ts'; import { parseEmailHmacKeyringEnvironment } from './growth-control.mts'; @@ -28,6 +30,8 @@ export interface ObservabilityOperatorDependencies { writeError(value: string): void; } const flags: Record = { + funnel: ['from', 'to'], + journey: ['contact'], health: ['from', 'to'], timeline: ['subject', 'cursor', 'limit'], detail: ['observation', 'include-identity'], @@ -77,11 +81,12 @@ function parseArguments(argv: readonly string[]) { if (!value || value.startsWith('--')) invalid(); args[key] = value; } - if (command === 'health') { + if (command === 'health' || command === 'funnel') { const from = date(args.from), to = date(args.to); if (to <= from || to.getTime() - from.getTime() > 31 * 86400000) invalid(); } + if (command === 'journey') uuid(args.contact); if (command === 'timeline') { uuid(args.subject); count(args.limit, 100, 100); @@ -146,6 +151,15 @@ export async function runGrowthObservability( db = deps.createDatabase(); let result: unknown; switch (command) { + case 'funnel': + result = await readGrowthFunnel(db, { + from: date(args.from), + to: date(args.to), + }); + break; + case 'journey': + result = await readContactJourney(db, args.contact); + break; case 'project-forms': result = await projectFormObservations(db, { enabled: true, diff --git a/scripts/growth-observability.spec.ts b/scripts/growth-observability.spec.ts index 03484a845..1fb4ce83e 100644 --- a/scripts/growth-observability.spec.ts +++ b/scripts/growth-observability.spec.ts @@ -16,6 +16,35 @@ function dependencies() { }; } describe('observation operator boundary', () => { + it.each([ + ['funnel', '--from', '2026-09-04', '--to', '2026-09-05'], + ['journey', '--contact', '11111111-1111-4111-8111-111111111111'], + ])('supports read-only %s without keyring or processing', async (...args) => { + const close = vi.fn(async () => undefined); + const deps = { + ...dependencies(), + createDatabase: vi.fn(() => ({ + execute: vi.fn(async () => ({ rows: [] })), + close, + transaction: vi.fn(), + })), + }; + expect(await runGrowthObservability(args, deps)).toBe(0); + expect(close).toHaveBeenCalledOnce(); + expect(deps.loadKeyring).not.toHaveBeenCalled(); + }); + it.each([ + ['funnel', '--from', '2026-01-01', '--to', '2026-09-05'], + ['funnel', '--from', '2026-09-05', '--to', '2026-09-04'], + ['journey', '--contact', 'private@example.invalid'], + ])( + 'rejects invalid report arguments before connecting: %s', + async (...args) => { + const deps = dependencies(); + expect(await runGrowthObservability(args, deps)).toBe(2); + expect(deps.createDatabase).not.toHaveBeenCalled(); + } + ); it('allows redacted reads with processing disabled and closes the executor', async () => { const execute = vi.fn(async () => ({ rows: [] })), close = vi.fn(async () => undefined);