diff --git a/apps/website/src/app/api/ingest/route.spec.ts b/apps/website/src/app/api/ingest/route.spec.ts index c357156ea..2f35a16b0 100644 --- a/apps/website/src/app/api/ingest/route.spec.ts +++ b/apps/website/src/app/api/ingest/route.spec.ts @@ -70,6 +70,66 @@ describe('/api/ingest', () => { expect(response.status).toBe(400); expect(response.headers.get('access-control-allow-origin')).toBe('*'); }); + + it.each([ + { event: 'tplane:invented', properties: { transport: 'custom' } }, + { event: 'tplane:postinstall', properties: {} }, + { event: 'tplane:stream_started', properties: {} }, + { event: 'tplane:stream_started', properties: 'secret' }, + { event: 'tplane:stream_started', properties: ['secret'] }, + { event: 'tplane:stream_started', properties: { transport: 1 } }, + { event: 'tplane:browser_chat_init', properties: {} }, + { event: 'tplane:stream_ended', properties: { transport: 'custom', durationMs: -1 } }, + ])('rejects malformed public payloads without capturing or echoing input', async (payload) => { + const response = await POST(new Request('https://threadplane.ai/api/ingest', { + method: 'POST', body: JSON.stringify({ distinctId: 'test', ...payload }), + }) as never); + expect(response.status).toBe(400); + expect(response.headers.get('access-control-allow-origin')).toBe('*'); + expect(await response.json()).toEqual({ error: 'Invalid event payload' }); + expect(capture).not.toHaveBeenCalled(); + }); + + it('accepts canonical runtime events while excluding arbitrary fields and person profiles', async () => { + const response = await POST(new Request('https://threadplane.ai/api/ingest', { + method: 'POST', body: JSON.stringify({ + distinctId: 'browser:test', event: 'tplane:stream_ended', + properties: { + transport: 'langgraph', surface: 'canonical_demo', durationMs: 120, + '0': 'private', command: 'private', body: 'private', token: 'private', + $set: { email: 'private' }, $ip: '1.2.3.4', $process_person_profile: true, + }, + }), + }) as never); + expect(response.status).toBe(202); + expect(capture).toHaveBeenCalledWith({ + distinctId: 'browser:test', event: 'tplane:stream_ended', + properties: { transport: 'langgraph', surface: 'canonical_demo', durationMs: 120, $ip: null, $process_person_profile: false }, + }); + }); + + it('rejects an oversized streamed body even without content-length', async () => { + const response = await POST(new Request('https://threadplane.ai/api/ingest', { + method: 'POST', body: JSON.stringify({ distinctId: 'test', event: 'tplane:browser_provided', properties: { body: 'x'.repeat(16_384) } }), + }) as never); + expect(response.status).toBe(413); + expect(response.headers.get('access-control-allow-origin')).toBe('*'); + expect(capture).not.toHaveBeenCalled(); + }); + + it('reports provider failure without logging the raw exception', async () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => undefined); + shutdown.mockRejectedValueOnce(new Error('private provider response')); + try { + const response = await POST(new Request('https://threadplane.ai/api/ingest', { + method: 'POST', body: JSON.stringify({ distinctId: 'test', event: 'tplane:browser_provided', properties: {} }), + }) as never); + expect(response.status).toBe(502); + expect(log).toHaveBeenCalledWith('[telemetry-ingest] capture failed'); + } finally { + log.mockRestore(); + } + }); }); /** diff --git a/apps/website/src/app/api/ingest/route.ts b/apps/website/src/app/api/ingest/route.ts index f105e5bf8..d14b8874d 100644 --- a/apps/website/src/app/api/ingest/route.ts +++ b/apps/website/src/app/api/ingest/route.ts @@ -1,8 +1,10 @@ import { PostHog } from 'posthog-node'; import { NextRequest, NextResponse } from 'next/server'; -import { normalizePostHogHost, toSafeAnalyticsString } from '@threadplane/telemetry/shared'; +import { normalizePostHogHost, parseTelemetryEvent, toSafeAnalyticsString } from '@threadplane/telemetry/shared'; +import { readBoundedBody } from '../_internal/read-bounded-body'; const PUBLIC_INGEST_KEY = 'phc_public_cacheplane_telemetry'; +const MAX_BODY_BYTES = 16_384; const CORS_HEADERS = { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'POST, OPTIONS', @@ -41,13 +43,13 @@ function readPayload(value: unknown): { if (payload.key !== undefined && payload.key !== PUBLIC_INGEST_KEY) return null; const distinctId = toSafeAnalyticsString(payload.distinctId, 200); - const event = toSafeAnalyticsString(payload.event, 100); - if (!distinctId || !event?.startsWith('tplane:')) return null; + const parsed = parseTelemetryEvent(payload.event, payload.properties); + if (!distinctId || !parsed) return null; return { distinctId, - event, - properties: isRecord(payload.properties) ? payload.properties : {}, + event: parsed.event, + properties: parsed.properties, }; } @@ -65,7 +67,9 @@ export function OPTIONS(): NextResponse { export async function POST(req: NextRequest) { let body: unknown; try { - body = await req.json(); + const rawBody = await readBoundedBody(req, MAX_BODY_BYTES); + if (rawBody === null) return jsonWithCors({ error: 'Invalid request body' }, { status: 413 }); + body = JSON.parse(rawBody); } catch { return jsonWithCors({ error: 'Invalid JSON' }, { status: 400 }); } @@ -98,8 +102,8 @@ export async function POST(req: NextRequest) { }); await posthog.shutdown(); return jsonWithCors({ ok: true }, { status: 202 }); - } catch (err) { - console.error('[telemetry-ingest] capture failed:', err); + } catch { + console.error('[telemetry-ingest] capture failed'); await posthog.shutdown().catch(() => undefined); return jsonWithCors( { error: 'Event ingest failed' }, diff --git a/docs/growth/README.md b/docs/growth/README.md index b1d66196c..14218d372 100644 --- a/docs/growth/README.md +++ b/docs/growth/README.md @@ -92,10 +92,13 @@ search reporting also have no in-repo recurring worker. The Growth funnel is an observation/activation report, not a complete sequential anonymous conversion funnel. A UTM is not a proven link from a social post to a -developer identity. Current PostHog report parsing needs separate improvement for -funnels and multiple trend series; do not treat unsupported or missing data as -proof of zero activity. An analytics contract failure does not itself prove a -lifecycle delivery failure. +developer identity. PostHog's Quick overview separates acquisition, docs, demo +and public runtime signals; install copy attempts do not measure npm installs. +The runtime dashboard includes demo usage and historical malformed events. +The weekly report separates additive daily series and marks funnels, unique +counts, breakdowns and missing results `Unavailable`. An analytics contract +failure does not itself prove a lifecycle delivery failure. See the +[dashboard inventory and measurement limits](../../tools/posthog/README.md#current-growth-dashboards). ## Contributor checks diff --git a/docs/gtm/taxonomy.md b/docs/gtm/taxonomy.md index 8f10ce6ff..7c89ecc54 100644 --- a/docs/gtm/taxonomy.md +++ b/docs/gtm/taxonomy.md @@ -36,7 +36,7 @@ The standard PostHog `$pageview` event is used as-is across all three surfaces. | `marketing:lead_form_submit` | Submit attempt (any surface) | | `marketing:lead_form_success` | Server 2xx | | `marketing:lead_form_fail` | Server non-2xx | -| `marketing:lead_qualified` | Server-side enrichment passes (qualified-lead def) | +| `marketing:lead_qualified` | Historical only: retired qualification emitter. Current evidence and authorization live in Growth. | | `marketing:newsletter_signup_submit` | Submit attempt | | `marketing:newsletter_signup_success` | Server 2xx | | `marketing:newsletter_signup_fail` | Failure | @@ -50,6 +50,18 @@ The standard PostHog `$pageview` event is used as-is across all three surfaces. | `blog:copy_code_click` | Copy-button click on a code block inside a blog post. Props: `surface: 'blog'`, `code_lang?`. | | `docs:tab_select` | MDX tab change | | `docs:sidebar_section_toggle` | Sidebar nav toggle | +| `docs:workspace_navigation` | Workspace capability navigation; `capability`, `category`, `from_capability`, `surface`. | +| `docs:workspace_mode_switched` | Workspace mode change; `capability`, `from_mode`, `to_mode`, `surface`. | +| `docs:workspace_runtime_action` | Explicit runtime action; `capability`, `action`, `state_before`, `outcome`, `surface`. | +| `docs:workspace_runtime_status_changed` | Runtime status transition; `capability`, `from_state`, `to_state`, optional `elapsed_ms`/`reason_code`, `surface`. | +| `marketing:stage_progress` | Recorded homepage stage progress; `surface`, `stage_event`, optional `beat`. Not a live developer runtime. | + +Current dashboards distinguish website intent, client-observed form acceptance, +and independent demo milestones. `hero_install` is a copy attempt recorded before +clipboard success, not an npm install. The former six-signal activation funnel +does not represent Growth's install/runtime activation and is no longer managed. +Actual install/runtime activation, enrichment, authorization and email outcomes +remain authoritative in Neon; see [Growth operations](../growth/README.md). ## Cockpit (activation surface) diff --git a/libs/telemetry/README.md b/libs/telemetry/README.md index 59f92bb5b..6ef7a3fa4 100644 --- a/libs/telemetry/README.md +++ b/libs/telemetry/README.md @@ -151,6 +151,25 @@ await captureEvent('tplane:runtime_instance_created', { The runtime adapter helpers exported from `@threadplane/telemetry/node` are convenience wrappers around the same explicit capture path. +The Node capture path and Threadplane's public ingest endpoint validate the seven +documented SDK event names at runtime. Runtime events require `transport`; +`tplane:browser_chat_init` requires `surface`. Property bags must be plain objects. +Only `transport`, `surface`, `requestType`, `provider`, `model`, `errorClass`, +`angularVersion`, `durationMs`, and `sample_weight` are forwarded. Strings are +nonempty labels of at most 128 characters without control characters; durations +must be finite numbers from 0 to 86,400,000 milliseconds, and sampling weights +must be finite numbers of at least 1, preserving reciprocal weights at low sample +rates. Unknown properties are dropped; invalid known metadata +rejects the event. Do not put user content or credentials in metadata labels. +Public submissions are untrusted observations, not verified product activity. + +`captureEvent()` returns `{ sent: false, reason: 'invalid' }` for invalid inputs. +The Node stream helpers accept an optional `transport`; valid legacy calls with +provider/model but no transport report `unknown`, without guessing an adapter. +Pass the transport explicitly for meaningful transport breakdowns. Generic +`captureEvent()` calls do not receive this fallback. These checks do not change +browser opt-in, custom sinks, or development-only Growth collection controls. + Set `TPLANE_TELEMETRY_INGEST_URL` to route events to an endpoint you control. The default endpoint is `https://threadplane.ai/api/ingest`. diff --git a/libs/telemetry/src/node/adapter.spec.ts b/libs/telemetry/src/node/adapter.spec.ts index 722960e72..2a0370223 100644 --- a/libs/telemetry/src/node/adapter.spec.ts +++ b/libs/telemetry/src/node/adapter.spec.ts @@ -79,4 +79,21 @@ describe('adapter helpers', () => { vi.mocked(captureEvent).mockRejectedValueOnce(new Error('network')); await expect(captureStreamStarted({ provider: 'x', model: 'y' })).resolves.toBeUndefined(); }); + + test.each([captureStreamStarted, captureStreamEnded, captureStreamErrored])('legacy stream helpers identify transport as unknown', async (capture) => { + await capture({ provider: 'openai', model: 'gpt-4', error: new Error('private') }); + expect(vi.mocked(captureEvent).mock.calls[0][1]).toMatchObject({ transport: 'unknown' }); + }); + + test.each([captureStreamStarted, captureStreamEnded, captureStreamErrored])('stream helpers preserve an explicit transport', async (capture) => { + await capture({ transport: 'ag-ui', provider: 'openai', model: 'gpt-4', error: new Error('private') } as never); + expect(vi.mocked(captureEvent).mock.calls[0][1]).toMatchObject({ transport: 'ag-ui' }); + }); + + test.each([null, 'private', 42, [], new Date(), {}, { provider: 'openai' }, { provider: '', model: 'gpt-4' }])('stream helpers do not manufacture events from malformed inputs', async (input) => { + await captureStreamStarted(input as never); + await captureStreamEnded(input as never); + await captureStreamErrored(input as never); + expect(captureEvent).not.toHaveBeenCalled(); + }); }); diff --git a/libs/telemetry/src/node/adapter.ts b/libs/telemetry/src/node/adapter.ts index 2f0c5261f..e2eee2c46 100644 --- a/libs/telemetry/src/node/adapter.ts +++ b/libs/telemetry/src/node/adapter.ts @@ -9,6 +9,8 @@ export interface RuntimeInstanceTelemetry { } export interface StreamTelemetry { + /** Runtime transport when known; legacy calls without it report `unknown`. */ + transport?: string; provider: string; model: string; durationMs?: number; @@ -25,6 +27,12 @@ async function safe(fn: () => Promise): Promise { try { await fn(); } catch { /* silent fail */ } } +function streamProperties(input: StreamTelemetry): Record | null { + if (!input || typeof input !== 'object' || Object.getPrototypeOf(input) !== Object.prototype) return null; + if (typeof input.provider !== 'string' || !input.provider.trim() || typeof input.model !== 'string' || !input.model.trim()) return null; + return { ...input, transport: input.transport === undefined ? 'unknown' : input.transport }; +} + export async function captureRuntimeInstanceCreated(input: RuntimeInstanceTelemetry): Promise { await safe(async () => { const { apiKey, ...rest } = input; @@ -38,19 +46,27 @@ export async function captureRuntimeRequestCreated(input: RuntimeRequestTelemetr } export async function captureStreamStarted(input: StreamTelemetry): Promise { - await safe(() => captureEvent('tplane:stream_started', { ...input })); + await safe(async () => { + const properties = streamProperties(input); + if (properties) await captureEvent('tplane:stream_started', properties); + }); } export async function captureStreamEnded(input: StreamTelemetry): Promise { - await safe(() => captureEvent('tplane:stream_ended', { ...input })); + await safe(async () => { + const properties = streamProperties(input); + if (properties) await captureEvent('tplane:stream_ended', properties); + }); } export async function captureStreamErrored( input: StreamTelemetry & { error: Error | unknown }, ): Promise { await safe(async () => { + const properties = streamProperties(input); + if (!properties) return; const { error, ...rest } = input; const errorClass = error instanceof Error ? error.constructor.name : 'Unknown'; - await captureEvent('tplane:stream_errored', { ...rest, errorClass }); + await captureEvent('tplane:stream_errored', { ...rest, transport: properties['transport'], errorClass }); }); } diff --git a/libs/telemetry/src/node/client.spec.ts b/libs/telemetry/src/node/client.spec.ts index f9366ba32..8ef07aee3 100644 --- a/libs/telemetry/src/node/client.spec.ts +++ b/libs/telemetry/src/node/client.spec.ts @@ -37,13 +37,13 @@ describe('node client', () => { test('uses the configured ingest endpoint', async () => { process.env.TPLANE_TELEMETRY_INGEST_URL = 'https://custom.example/api/ingest'; - await captureEvent('tplane:stream_started', {}); + await captureEvent('tplane:stream_started', { transport: 'custom' }); expect(fetchMock.mock.calls[0][0]).toBe('https://custom.example/api/ingest'); }); test('defaults to the Threadplane ingest proxy', async () => { delete process.env.TPLANE_TELEMETRY_INGEST_URL; - await captureEvent('tplane:stream_started', {}); + await captureEvent('tplane:stream_started', { transport: 'custom' }); expect(fetchMock.mock.calls[0][0]).toBe('https://threadplane.ai/api/ingest'); }); @@ -67,7 +67,7 @@ describe('node client', () => { test('reports failed sends instead of throwing', async () => { fetchMock.mockRejectedValueOnce(new Error('network')); - await expect(captureEvent('tplane:stream_errored', {})).resolves.toEqual({ + await expect(captureEvent('tplane:stream_errored', { transport: 'custom' })).resolves.toEqual({ sent: false, reason: 'failed', }); @@ -75,7 +75,25 @@ describe('node client', () => { test('invalid sample rate falls back to sending', async () => { process.env.TPLANE_TELEMETRY_SAMPLE_RATE = 'not-a-number'; - await expect(captureEvent('tplane:stream_started', {})).resolves.toEqual({ sent: true }); + await expect(captureEvent('tplane:stream_started', { transport: 'custom' })).resolves.toEqual({ sent: true }); expect(fetchMock).toHaveBeenCalledOnce(); }); + + test.each([{}, null, 'secret', [], { transport: 42 }, { transport: 'custom', model: { secret: true } }])( + 'silently rejects malformed runtime properties before sending', async (properties) => { + await expect(captureEvent('tplane:stream_started', properties as never)).resolves.toEqual({ sent: false, reason: 'invalid' }); + expect(fetchMock).not.toHaveBeenCalled(); + }, + ); + + test('rejects unknown runtime event names', async () => { + await expect(captureEvent('tplane:invented' as never, { transport: 'custom' })).resolves.toEqual({ sent: false, reason: 'invalid' }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + test('does not forward arbitrary properties or caller-supplied sampling weight', async () => { + await captureEvent('tplane:stream_started', { transport: 'custom', command: 'secret', token: 'secret', sample_weight: 9 }); + const body = JSON.parse(String(fetchMock.mock.calls[0][1].body)); + expect(body.properties).toEqual({ transport: 'custom', sample_weight: 1 }); + }); }); diff --git a/libs/telemetry/src/node/client.ts b/libs/telemetry/src/node/client.ts index c06363881..f43d35468 100644 --- a/libs/telemetry/src/node/client.ts +++ b/libs/telemetry/src/node/client.ts @@ -2,6 +2,7 @@ import { getAnonId } from '../shared/anon-id.js'; import { isTelemetryDisabled } from '../shared/env.js'; import { shouldSample } from '../shared/sample.js'; import type { ThreadplaneNodeEvent } from '../shared/events.js'; +import { parseTelemetryEvent } from '../shared/ingest.js'; import { isProgrammaticallyDisabled } from './disable.js'; const DEFAULT_INGEST = 'https://threadplane.ai/api/ingest'; @@ -12,7 +13,7 @@ const PUBLIC_INGEST_KEY = 'phc_public_cacheplane_telemetry'; export type CaptureResult = | { sent: true } - | { sent: false; reason: 'disabled' | 'sampled' | 'failed' }; + | { sent: false; reason: 'disabled' | 'sampled' | 'failed' | 'invalid' }; function getSampleRate(env: NodeJS.ProcessEnv = process.env): number { const parsed = Number(env.TPLANE_TELEMETRY_SAMPLE_RATE ?? '1'); @@ -42,18 +43,22 @@ export async function captureEvent( ): Promise { if (isTelemetryDisabled() || isProgrammaticallyDisabled()) return { sent: false, reason: 'disabled' }; + const parsed = parseTelemetryEvent(event, properties); + if (!parsed || parsed.event.startsWith('tplane:browser_')) return { sent: false, reason: 'invalid' }; const rate = getSampleRate(); const anonId = getAnonId(); if (!shouldSample(rate, anonId)) return { sent: false, reason: 'sampled' }; + const payload = parseTelemetryEvent(parsed.event, { + ...parsed.properties, + sample_weight: rate > 0 ? 1 / Math.min(1, rate) : 1, + }); + if (!payload) return { sent: false, reason: 'invalid' }; try { await postJson(process.env.TPLANE_TELEMETRY_INGEST_URL ?? DEFAULT_INGEST, { key: PUBLIC_INGEST_KEY, distinctId: anonId, - event, - properties: { - ...properties, - sample_weight: rate > 0 ? 1 / Math.min(1, rate) : 1, - }, + event: payload.event, + properties: payload.properties, }); return { sent: true }; } catch { diff --git a/libs/telemetry/src/shared/ingest.spec.ts b/libs/telemetry/src/shared/ingest.spec.ts new file mode 100644 index 000000000..9ef292daa --- /dev/null +++ b/libs/telemetry/src/shared/ingest.spec.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { parseTelemetryEvent } from './ingest'; + +describe('public telemetry event parsing', () => { + it.each([ + 'tplane:runtime_instance_created', 'tplane:runtime_request_created', + 'tplane:stream_started', 'tplane:stream_ended', 'tplane:stream_errored', + ])('accepts %s with its required transport', (event) => { + expect(parseTelemetryEvent(event, { transport: 'langgraph', surface: 'canonical_demo' })) + .toEqual({ event, properties: { transport: 'langgraph', surface: 'canonical_demo' } }); + expect(parseTelemetryEvent(event, {})).toBeNull(); + }); + + it('requires a surface for chat init but permits a property-free provider event', () => { + expect(parseTelemetryEvent('tplane:browser_chat_init', {})).toBeNull(); + expect(parseTelemetryEvent('tplane:browser_chat_init', { surface: 'canonical_demo' })).not.toBeNull(); + expect(parseTelemetryEvent('tplane:browser_provided', {})).not.toBeNull(); + }); + + it.each(['tplane:made_up', 'tplane:postinstall', 'other', null, 1])('rejects unknown event %s', (event) => { + expect(parseTelemetryEvent(event, { transport: 'custom' })).toBeNull(); + }); + + it.each([null, undefined, 'langgraph', 42, [], new Date(), Object.create({ transport: 'custom' })])( + 'rejects malformed property containers', (properties) => { + expect(parseTelemetryEvent('tplane:stream_started', properties)).toBeNull(); + }, + ); + + it('keeps only bounded primitive metadata, excluding arbitrary and sensitive fields', () => { + const result = parseTelemetryEvent('tplane:stream_ended', { + transport: 'custom', surface: 'canonical_demo', requestType: 'submit', + provider: 'openai', model: 'gpt-4', angularVersion: '22.0.0', + durationMs: 124, sample_weight: 2, errorClass: 'TypeError', + '0': 'secret', command: 'secret', body: { secret: true }, token: 'secret', + apiKey: 'secret', errorMessage: 'secret', arbitrary: true, $set: { email: 'secret' }, + $ip: '1.2.3.4', $process_person_profile: true, + }); + expect(result?.properties).toEqual({ + transport: 'custom', surface: 'canonical_demo', requestType: 'submit', + provider: 'openai', model: 'gpt-4', angularVersion: '22.0.0', + durationMs: 124, sample_weight: 2, errorClass: 'TypeError', + }); + }); + + it.each([ + { transport: 1 }, { transport: '' }, { transport: ' ' }, + { model: { secret: true } }, { provider: ['openai'] }, + { surface: 'x'.repeat(129) }, { errorClass: 'TypeError\nsecret' }, + { durationMs: -1 }, { durationMs: Infinity }, { durationMs: '42' }, + { durationMs: 86_400_001 }, { sample_weight: 0 }, { sample_weight: NaN }, + ])('rejects invalid known metadata without coercion', (properties) => { + expect(parseTelemetryEvent('tplane:stream_ended', { transport: 'custom', ...properties })).toBeNull(); + }); + + it('preserves finite reciprocal weights for sampling rates below one in a million', () => { + expect(parseTelemetryEvent('tplane:stream_started', { transport: 'custom', sample_weight: 10_000_000 })) + .toEqual({ event: 'tplane:stream_started', properties: { transport: 'custom', sample_weight: 10_000_000 } }); + expect(parseTelemetryEvent('tplane:stream_started', { transport: 'custom', sample_weight: Infinity })).toBeNull(); + }); + + it('does not execute property accessors or throw for hostile runtime inputs', () => { + const getter = () => { throw new Error('secret'); }; + const properties = Object.defineProperty({ transport: 'custom' }, 'model', { get: getter }); + expect(parseTelemetryEvent('tplane:stream_started', properties)).toBeNull(); + const proxy = new Proxy({}, { getPrototypeOf: getter }); + expect(parseTelemetryEvent('tplane:stream_started', proxy)).toBeNull(); + }); +}); diff --git a/libs/telemetry/src/shared/ingest.ts b/libs/telemetry/src/shared/ingest.ts new file mode 100644 index 000000000..7092efb4f --- /dev/null +++ b/libs/telemetry/src/shared/ingest.ts @@ -0,0 +1,61 @@ +import type { ThreadplaneEvent } from './events.js'; + +const EVENTS: ReadonlySet = new Set([ + 'tplane:runtime_instance_created', + 'tplane:runtime_request_created', + 'tplane:stream_started', + 'tplane:stream_ended', + 'tplane:stream_errored', + 'tplane:browser_provided', + 'tplane:browser_chat_init', +]); + +const STRING_PROPERTIES = new Set([ + 'transport', 'surface', 'requestType', 'provider', 'model', 'errorClass', 'angularVersion', +]); + +/** A validated public SDK event containing only bounded metadata. */ +export interface ParsedTelemetryEvent { + event: ThreadplaneEvent; + properties: Record; +} + +/** + * Validate untrusted public SDK events and copy only allowed primitive metadata. + * Unknown properties are omitted; malformed known properties reject the event. + * Returns null without exposing payload data or executing property getters. + */ +export function parseTelemetryEvent(event: unknown, properties: unknown): ParsedTelemetryEvent | null { + try { + if (typeof event !== 'string' || !EVENTS.has(event)) return null; + if (properties === null || typeof properties !== 'object' || Array.isArray(properties)) return null; + const prototype = Object.getPrototypeOf(properties); + if (prototype !== Object.prototype && prototype !== null) return null; + + const result: Record = {}; + for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(properties))) { + if (!STRING_PROPERTIES.has(key) && key !== 'durationMs' && key !== 'sample_weight') continue; + if (!('value' in descriptor)) return null; + const value: unknown = descriptor.value; + if (value === undefined) continue; + if (STRING_PROPERTIES.has(key)) { + // eslint-disable-next-line no-control-regex -- Control characters are deliberately rejected from public metadata. + if (typeof value !== 'string' || /[\u0000-\u001f\u007f]/u.test(value)) return null; + const label = value.trim(); + if (!label || label.length > 128) return null; + result[key] = label; + } else { + if (typeof value !== 'number' || !Number.isFinite(value)) return null; + if (key === 'durationMs' && (value < 0 || value > 86_400_000)) return null; + if (key === 'sample_weight' && value < 1) return null; + result[key] = value; + } + } + if (event === 'tplane:browser_chat_init' && !result['surface']) return null; + if (event !== 'tplane:browser_provided' && event !== 'tplane:browser_chat_init' && !result['transport']) return null; + return { event: event as ThreadplaneEvent, properties: result }; + } catch { + // JavaScript callers can pass hostile proxies as well as plain JSON data. + return null; + } +} diff --git a/libs/telemetry/src/shared/public-api.ts b/libs/telemetry/src/shared/public-api.ts index 919d9df94..283e2dfea 100644 --- a/libs/telemetry/src/shared/public-api.ts +++ b/libs/telemetry/src/shared/public-api.ts @@ -1,4 +1,6 @@ export type { ThreadplaneEvent, ThreadplaneNodeEvent, ThreadplaneBrowserEvent } from './events'; +export { parseTelemetryEvent } from './ingest'; +export type { ParsedTelemetryEvent } from './ingest'; export { getEmailDomain, getSourcePage, diff --git a/tools/posthog/README.md b/tools/posthog/README.md index 0f821bae1..54fa055b4 100644 --- a/tools/posthog/README.md +++ b/tools/posthog/README.md @@ -6,6 +6,29 @@ Part of [Growth architecture and operations](../../docs/growth/README.md). PostHog is configured via a Public-API-driven sync script — not through the PostHog UI. Every dashboard, insight, and cohort the GTM motion depends on is a JSON file in this directory. The sync tool reconciles JSON ↔ PostHog. Git is the source of truth. +## Current Growth dashboards + +| Dashboard | Answers | +| --- | --- | +| [Growth · Quick overview](https://us.posthog.com/project/406826/dashboard/2073577) | Which acquisition, docs, demo and public runtime signals are present? Are runtime events missing transport? | +| [Growth · Acquisition and demo engagement](https://us.posthog.com/project/406826/dashboard/1582272) | Which landing pages, install-dialog actions, forms, docs capabilities and independent demo milestones receive activity? | +| [Growth · Runtime diagnostics](https://us.posthog.com/project/406826/dashboard/1592941) | Which transports produce runtime instances, requests, stream starts, ends and errors? | + +These are event counts, not a joined developer conversion funnel. Install command +copy attempts are recorded before the clipboard operation; they are not completed +npm installs. Form acceptance is observed by the browser, not proof of a unique, +qualified or email-authorized contact. Runtime counts include the canonical demo +and public, unauthenticated SDK telemetry. Missing and `unknown` transport are not +real adapters. Historical malformed events remain visible. + +Install observations, development announcements, activation, enrichment, email +eligibility and delivery outcomes remain authoritative in Neon. Use +`npm run growth:report -- funnel --from --to ` and +`npm run growth:report -- journey --contact ` through the +[Growth operations guide](../../docs/growth/README.md). No Neon export is added +by these dashboards. The untracked legacy package dashboard uses the retired +`tplane:postinstall` event and does not measure the current install flow. + ## Directory layout ``` @@ -70,26 +93,27 @@ Env vars (see `.env.example` at repo root): { "slug": "developer-funnel", // local id, stable across syncs "posthog_id": null, // assigned on first sync; do not edit - "name": "GTM · Developer funnel", - "description": "Pageview → install → cockpit activation.", - "tags": ["gtm", "developer-track"], + "name": "Growth · Acquisition and demo engagement", + "description": "Independent acquisition and demo signals; lifecycle outcomes live in Neon.", + "tags": ["gtm", "growth", "acquisition"], "tiles": [ { "insight": "pageviews-by-landing" }, - { "insight": "six-signal-activation-funnel" } + { "insight": "growth-install-intent" } ] } ``` ```jsonc -// tools/posthog/insights/six-signal-activation-funnel.json +// Example trend definition { - "slug": "six-signal-activation-funnel", + "slug": "install-command-clicks", "posthog_id": null, - "kind": "funnel", - "window_minutes": 30, - "steps": [ - { "event": "cockpit:install_command_copied" }, - { "event": "cockpit:transport_connected" } + "name": "Install command copy attempts", + "kind": "trends", + "interval": "day", + "events": [ + { "event": "marketing:cta_click", "math": "total", + "properties": [{ "key": "cta_id", "value": "hero_install" }] } ] } ``` @@ -102,15 +126,23 @@ Event names must match [`docs/gtm/taxonomy.md`](../../docs/gtm/taxonomy.md). The - `taxonomy.spec.ts` and `telemetry-contract.spec.ts` guard committed dashboard JSON against undocumented events, unsupported breakdowns, unsupported filters, runtime dashboard coverage drift, and forbidden sensitive runtime fields. - `npm run posthog:quality -- --days 7 --limit-per-event 25` samples recent live PostHog events and validates observed payloads against the same contract. It exits non-zero for missing required properties or forbidden sensitive properties, and prints warnings for non-contract fields. -- `npm run posthog:quality -- --days 7 --limit-per-event 100 --require-critical-coverage` also requires recent samples for critical install and runtime events. The scheduled `PostHog telemetry quality` workflow runs this thresholded check daily and supports manual dispatch. -- The live workflow requires Actions secrets named `POSTHOG_PERSONAL_API_KEY` and `POSTHOG_PROJECT_ID`. +- `npm run posthog:quality -- --days 7 --limit-per-event 100 --require-critical-coverage` also requires recent samples for critical browser and runtime events. The scheduled `PostHog telemetry quality` workflow runs this thresholded seven-day check daily and supports manual dispatch. +- The live workflow requires Actions secrets named `POSTHOG_PERSONAL_API_KEY_READONLY` and `POSTHOG_PROJECT_ID`. +- Counts are bounded samples per event name, not a traffic census. A zero count does not establish healthy collection. Use `--days 1` for a current-day comparison without replacing the seven-day audit. Historical malformed events can continue to fail that audit after a fix ships; do not delete them or infer a lifecycle outage from this alone. +- Public SDK admission uses `libs/telemetry/src/shared/ingest.ts` at Node capture and website ingest: known events, required runtime transport/browser-chat surface, bounded primitive metadata, and a 16 KiB HTTP body limit. Unknown properties are dropped and malformed known values rejected. This validates shape, not caller identity. Legacy Node stream helpers report `unknown` when transport is omitted. ## Sync semantics - **`--plan`** — diff against PostHog, no writes. Outputs `[create] [update] [orphan]` per artifact. CI runs this on every PR that affects `posthog-tools`. -- **`--apply`** — idempotent upsert via PATCH. Re-running with no JSON change is a no-op (PostHog dedupes). +- **`--apply`** — upsert via PATCH. Re-running preserves object IDs but still writes managed metadata. Membership reconciliation detaches stale tiles from managed dashboards and preserves memberships in unrelated dashboards. Wiring failures make the command fail. - **`--apply --delete-orphans`** — explicit deletion of remote artifacts that have no local JSON. Never automatic. -- **`posthog_id` writeback** — first successful create writes the assigned PostHog id back into the JSON. Commit the writeback as `chore(posthog): writeback ids for `. +- **`posthog_id` writeback** — first successful create writes the assigned PostHog id back into the JSON. Include these IDs with the finished dashboard change. + +The weekly report fetches details for repository-managed dashboards and separates +additive daily trend series over the last 28 complete +UTC days, excluding today's partial bucket. Funnels, unique counts, +breakdowns and missing/stale/incomplete results render as `Unavailable` with a reason; +they are never silently converted to zero. Inspect those insights in PostHog. ## Renaming an artifact diff --git a/tools/posthog/dashboards/developer-funnel.json b/tools/posthog/dashboards/developer-funnel.json index 34db728cc..36c14416d 100644 --- a/tools/posthog/dashboards/developer-funnel.json +++ b/tools/posthog/dashboards/developer-funnel.json @@ -1,25 +1,31 @@ { "slug": "developer-funnel", "posthog_id": 1582272, - "name": "GTM · Developer funnel", - "description": "Pageview → install → cockpit activation. Source: gtm.md §4.", + "name": "Growth \u00b7 Acquisition and demo engagement", + "description": "Website intent, client-observed form acceptance and independent demo milestones. Not a sequential install-to-email funnel. Actual install/runtime activation, enrichment and outreach outcomes are in Neon; use growth:report. https://github.com/cacheplane/angular-agent-framework/blob/main/docs/growth/README.md", "tags": [ "gtm", - "developer-track", - "phase-1" + "growth", + "acquisition" ], "tiles": [ { "insight": "pageviews-by-landing" }, + { + "insight": "growth-install-intent" + }, { "insight": "install-command-clicks" }, { - "insight": "cockpit-recipe-completion" + "insight": "growth-form-acceptance" }, { - "insight": "activation-funnel" + "insight": "growth-docs-engagement" + }, + { + "insight": "cockpit-recipe-completion" } ] } diff --git a/tools/posthog/dashboards/growth-overview.json b/tools/posthog/dashboards/growth-overview.json new file mode 100644 index 000000000..c90978b20 --- /dev/null +++ b/tools/posthog/dashboards/growth-overview.json @@ -0,0 +1,34 @@ +{ + "slug": "growth-overview", + "posthog_id": 2073577, + "name": "Growth · Quick overview", + "description": "V1 acquisition and observed usage, with explicit data-quality context. Copy attempts are not installs; demo milestones are not activation. Neon owns actual installs, enrichment, authorization and email outcomes: npm run growth:report -- funnel / journey. No automatic person linkage is implied. https://github.com/cacheplane/angular-agent-framework/blob/main/docs/growth/README.md", + "tags": [ + "gtm", + "growth", + "overview" + ], + "tiles": [ + { + "insight": "pageviews-by-landing" + }, + { + "insight": "growth-install-intent" + }, + { + "insight": "growth-form-acceptance" + }, + { + "insight": "growth-docs-engagement" + }, + { + "insight": "cockpit-recipe-completion" + }, + { + "insight": "growth-runtime-surfaces" + }, + { + "insight": "growth-telemetry-missing-transport" + } + ] +} diff --git a/tools/posthog/dashboards/runtime-telemetry.json b/tools/posthog/dashboards/runtime-telemetry.json index f9389f14b..0345d899c 100644 --- a/tools/posthog/dashboards/runtime-telemetry.json +++ b/tools/posthog/dashboards/runtime-telemetry.json @@ -1,12 +1,12 @@ { "slug": "runtime-telemetry", "posthog_id": 1592941, - "name": "GTM · Runtime telemetry", - "description": "Explicit Node runtime and opt-in browser telemetry for @threadplane/* adapters.", + "name": "Growth \u00b7 Runtime diagnostics", + "description": "Explicit runtime telemetry, including canonical demos. Counts are operational signals, not authenticated developer activation. Missing or unknown transport remains visible; see the Growth quick overview for malformed-event counts.", "tags": [ "gtm", - "runtime-telemetry", - "phase-1" + "growth", + "runtime-telemetry" ], "tiles": [ { diff --git a/tools/posthog/insights/activation-funnel.json b/tools/posthog/insights/activation-funnel.json deleted file mode 100644 index d8555f70c..000000000 --- a/tools/posthog/insights/activation-funnel.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "slug": "activation-funnel", - "posthog_id": 8638740, - "kind": "funnel", - "name": "Activation funnel (30-min window)", - "interval": "day", - "window_minutes": 30, - "steps": [ - { - "event": "cockpit:chat_first_message" - }, - { - "event": "cockpit:transport_connected" - }, - { - "event": "cockpit:thread_persisted" - }, - { - "event": "cockpit:interrupt_handled" - }, - { - "event": "cockpit:generative_component_rendered" - } - ], - "date_from": "-30d" -} diff --git a/tools/posthog/insights/cockpit-recipe-completion.json b/tools/posthog/insights/cockpit-recipe-completion.json index 3d2cb2219..e3f21e039 100644 --- a/tools/posthog/insights/cockpit-recipe-completion.json +++ b/tools/posthog/insights/cockpit-recipe-completion.json @@ -2,17 +2,30 @@ "slug": "cockpit-recipe-completion", "posthog_id": 8587348, "kind": "trends", - "name": "Cockpit recipe completion", + "name": "Demo milestones \u2014 independent counts", "events": [ { - "event": "cockpit:recipe_opened", - "math": "total" + "event": "cockpit:transport_connected", + "math": "total", + "name": "Transport connected" }, { - "event": "cockpit:chat_first_message", - "math": "total" + "event": "cockpit:thread_persisted", + "math": "total", + "name": "Thread persisted" + }, + { + "event": "cockpit:interrupt_handled", + "math": "total", + "name": "Approval handled" + }, + { + "event": "cockpit:generative_component_rendered", + "math": "total", + "name": "Component rendered" } ], "interval": "day", - "date_from": "-30d" + "date_from": "-30d", + "description": "Observed demo connections, persistence, approvals and rendering. Independent event totals, not a sequential funnel, unique developers or install activation." } diff --git a/tools/posthog/insights/growth-docs-engagement.json b/tools/posthog/insights/growth-docs-engagement.json new file mode 100644 index 000000000..4bb294137 --- /dev/null +++ b/tools/posthog/insights/growth-docs-engagement.json @@ -0,0 +1,23 @@ +{ + "slug": "growth-docs-engagement", + "posthog_id": 11684147, + "kind": "trends", + "name": "Docs workspace engagement", + "description": "Workspace navigation and mode changes. Browsing examples is separate from development-runtime activation.", + "events": [ + { + "event": "docs:workspace_navigation", + "name": "Example navigation", + "math": "total" + }, + { + "event": "docs:workspace_mode_switched", + "name": "Mode changed", + "math": "total" + } + ], + "breakdown": "capability", + "breakdown_limit": 10, + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/growth-form-acceptance.json b/tools/posthog/insights/growth-form-acceptance.json new file mode 100644 index 000000000..7fd57ce2c --- /dev/null +++ b/tools/posthog/insights/growth-form-acceptance.json @@ -0,0 +1,26 @@ +{ + "slug": "growth-form-acceptance", + "posthog_id": 11684148, + "kind": "trends", + "name": "Form acceptance signals", + "description": "Client-observed successful form responses. These are submission events, not unique contacts, qualified accounts, email sends or authorization state. Neon is authoritative.", + "events": [ + { + "event": "marketing:lead_form_success", + "name": "Contact request accepted", + "math": "total" + }, + { + "event": "marketing:whitepaper_signup_success", + "name": "Guide request accepted", + "math": "total" + }, + { + "event": "marketing:newsletter_signup_success", + "name": "Newsletter request accepted", + "math": "total" + } + ], + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/growth-install-intent.json b/tools/posthog/insights/growth-install-intent.json new file mode 100644 index 000000000..d3a4f7304 --- /dev/null +++ b/tools/posthog/insights/growth-install-intent.json @@ -0,0 +1,35 @@ +{ + "slug": "growth-install-intent", + "posthog_id": 11684149, + "kind": "trends", + "name": "Install dialog opens and copy attempts", + "description": "Website intent only. Opening the dialog or clicking copy does not prove an npm install.", + "events": [ + { + "event": "marketing:cta_click", + "name": "Install dialog opened", + "math": "total", + "properties": [ + { + "key": "cta_id", + "value": "hero_install_open", + "operator": "exact" + } + ] + }, + { + "event": "marketing:cta_click", + "name": "Copy attempted", + "math": "total", + "properties": [ + { + "key": "cta_id", + "value": "hero_install", + "operator": "exact" + } + ] + } + ], + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/growth-runtime-surfaces.json b/tools/posthog/insights/growth-runtime-surfaces.json new file mode 100644 index 000000000..fb168d975 --- /dev/null +++ b/tools/posthog/insights/growth-runtime-surfaces.json @@ -0,0 +1,17 @@ +{ + "slug": "growth-runtime-surfaces", + "posthog_id": 11684150, + "kind": "trends", + "name": "Observed runtime requests by surface", + "description": "Public opt-in telemetry; includes our canonical demo. Not authenticated customer usage or install/runtime activation. Missing and unknown surfaces remain visible.", + "events": [ + { + "event": "tplane:runtime_request_created", + "math": "total" + } + ], + "breakdown": "surface", + "breakdown_limit": 10, + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/growth-telemetry-missing-transport.json b/tools/posthog/insights/growth-telemetry-missing-transport.json new file mode 100644 index 000000000..63c8fbd9c --- /dev/null +++ b/tools/posthog/insights/growth-telemetry-missing-transport.json @@ -0,0 +1,66 @@ +{ + "slug": "growth-telemetry-missing-transport", + "posthog_id": 11684151, + "kind": "trends", + "name": "Telemetry quality — missing transport", + "description": "Historical malformed runtime events remain visible. Admission rejects new malformed payloads after deployment. This is a data-quality count, not application errors.", + "events": [ + { + "event": "tplane:runtime_instance_created", + "math": "total", + "properties": [ + { + "key": "transport", + "value": "", + "operator": "is_not_set" + } + ] + }, + { + "event": "tplane:runtime_request_created", + "math": "total", + "properties": [ + { + "key": "transport", + "value": "", + "operator": "is_not_set" + } + ] + }, + { + "event": "tplane:stream_started", + "math": "total", + "properties": [ + { + "key": "transport", + "value": "", + "operator": "is_not_set" + } + ] + }, + { + "event": "tplane:stream_ended", + "math": "total", + "properties": [ + { + "key": "transport", + "value": "", + "operator": "is_not_set" + } + ] + }, + { + "event": "tplane:stream_errored", + "math": "total", + "properties": [ + { + "key": "transport", + "value": "", + "operator": "is_not_set" + } + ] + } + ], + "interval": "day", + "date_from": "-30d" +} diff --git a/tools/posthog/insights/install-command-clicks.json b/tools/posthog/insights/install-command-clicks.json index 03e4de8fd..0af25cb3a 100644 --- a/tools/posthog/insights/install-command-clicks.json +++ b/tools/posthog/insights/install-command-clicks.json @@ -2,15 +2,15 @@ "slug": "install-command-clicks", "posthog_id": 8587349, "kind": "trends", - "name": "Install command clicks", + "name": "Install command copy attempts", "events": [ { "event": "marketing:cta_click", "math": "total", "properties": [ { - "key": "track", - "value": "developer", + "key": "cta_id", + "value": "hero_install", "operator": "exact" } ] @@ -19,5 +19,6 @@ "breakdown": "cta_id", "breakdown_limit": 10, "interval": "day", - "date_from": "-30d" + "date_from": "-30d", + "description": "Clicks on Copy install command (hero_install). This event fires before clipboard success; it is intent, not an npm install." } diff --git a/tools/posthog/insights/runtime-instances-by-transport.json b/tools/posthog/insights/runtime-instances-by-transport.json index 78cea7df6..eb3e0ccee 100644 --- a/tools/posthog/insights/runtime-instances-by-transport.json +++ b/tools/posthog/insights/runtime-instances-by-transport.json @@ -12,5 +12,6 @@ "breakdown": "transport", "breakdown_limit": 10, "interval": "day", - "date_from": "-30d" + "date_from": "-30d", + "description": "Public runtime telemetry including canonical demo traffic. Independent event counts; not npm installs or activated developers. Missing/unknown transport is not a real adapter." } diff --git a/tools/posthog/insights/runtime-requests-by-transport.json b/tools/posthog/insights/runtime-requests-by-transport.json index fe3461910..36506876a 100644 --- a/tools/posthog/insights/runtime-requests-by-transport.json +++ b/tools/posthog/insights/runtime-requests-by-transport.json @@ -12,5 +12,6 @@ "breakdown": "transport", "breakdown_limit": 10, "interval": "day", - "date_from": "-30d" + "date_from": "-30d", + "description": "Public runtime telemetry including canonical demo traffic. Independent event counts; not npm installs or activated developers. Missing/unknown transport is not a real adapter." } diff --git a/tools/posthog/insights/runtime-stream-ends-by-transport.json b/tools/posthog/insights/runtime-stream-ends-by-transport.json index cf87ed280..35f2db3ff 100644 --- a/tools/posthog/insights/runtime-stream-ends-by-transport.json +++ b/tools/posthog/insights/runtime-stream-ends-by-transport.json @@ -12,5 +12,6 @@ "breakdown": "transport", "breakdown_limit": 10, "interval": "day", - "date_from": "-30d" + "date_from": "-30d", + "description": "Public runtime telemetry including canonical demo traffic. Independent event counts; not npm installs or activated developers. Missing/unknown transport is not a real adapter." } diff --git a/tools/posthog/insights/runtime-stream-errors-by-transport.json b/tools/posthog/insights/runtime-stream-errors-by-transport.json index 52c5997b7..0fedd0018 100644 --- a/tools/posthog/insights/runtime-stream-errors-by-transport.json +++ b/tools/posthog/insights/runtime-stream-errors-by-transport.json @@ -12,5 +12,6 @@ "breakdown": "transport", "breakdown_limit": 10, "interval": "day", - "date_from": "-30d" + "date_from": "-30d", + "description": "Public runtime telemetry including canonical demo traffic. Independent event counts; not npm installs or activated developers. Missing/unknown transport is not a real adapter." } diff --git a/tools/posthog/insights/runtime-stream-starts-by-transport.json b/tools/posthog/insights/runtime-stream-starts-by-transport.json index 8435933aa..39339d1f9 100644 --- a/tools/posthog/insights/runtime-stream-starts-by-transport.json +++ b/tools/posthog/insights/runtime-stream-starts-by-transport.json @@ -12,5 +12,6 @@ "breakdown": "transport", "breakdown_limit": 10, "interval": "day", - "date_from": "-30d" + "date_from": "-30d", + "description": "Public runtime telemetry including canonical demo traffic. Independent event counts; not npm installs or activated developers. Missing/unknown transport is not a real adapter." } diff --git a/tools/posthog/live-quality.spec.ts b/tools/posthog/live-quality.spec.ts index 0f2c5eae0..594660176 100644 --- a/tools/posthog/live-quality.spec.ts +++ b/tools/posthog/live-quality.spec.ts @@ -116,6 +116,9 @@ test('formatLiveQualityReport summarizes clean coverage and warnings', () => { assert.match(report, /\| tplane:stream_ended \| 0 \|/); assert.match(report, /Warnings/); assert.match(report, /unexpected/); + assert.match(report, /Sampled observations: 1/); + assert.match(report, /bounded sample, not total traffic/); + assert.match(report, /Zero samples do not establish healthy collection/); }); test('analyzeTelemetryCoverage flags required events with no recent samples', () => { diff --git a/tools/posthog/live-quality.ts b/tools/posthog/live-quality.ts index 5b9c30a0b..d0690ab0b 100644 --- a/tools/posthog/live-quality.ts +++ b/tools/posthog/live-quality.ts @@ -217,12 +217,14 @@ export function formatLiveQualityReport({ checkedEvents, coverageRequirements = [], days, + limitPerEvent, events, findings, }: { checkedEvents: readonly string[]; coverageRequirements?: readonly LiveCoverageRequirement[]; days: number; + limitPerEvent?: number; events: readonly LiveTelemetryEvent[]; findings: readonly LiveQualityFinding[]; }): string { @@ -237,6 +239,9 @@ export function formatLiveQualityReport({ `Live telemetry quality — last ${days} ${days === 1 ? 'day' : 'days'}` ); lines.push(''); + lines.push(`Sampled observations: ${events.length}. This is a bounded sample, not total traffic${limitPerEvent === undefined ? '.' : ` (up to ${limitPerEvent} events per event name).`}`); + lines.push('Zero samples do not establish healthy collection. Error and warning counts below count findings, not distinct events.'); + lines.push(''); if (coverageRequirements.length > 0) { lines.push('| Event | Sampled events | Required minimum |'); lines.push('|-------|---------------:|-----------------:|'); @@ -350,6 +355,7 @@ async function main(): Promise { checkedEvents, coverageRequirements: options.coverageRequirements, days: options.days, + limitPerEvent: options.limitPerEvent, events, findings, }) diff --git a/tools/posthog/report.spec.ts b/tools/posthog/report.spec.ts index 4e6235dd8..51f1ff562 100644 --- a/tools/posthog/report.spec.ts +++ b/tools/posthog/report.spec.ts @@ -1,6 +1,117 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { sparkline, formatDeltaCell, renderReport } from './report.js'; +import { sparkline, formatDeltaCell, renderReport, insightReportRows, generateReport } from './report.js'; + +const asOf = new Date('2026-09-07T17:45:00Z'); +const dayMs = 86_400_000; +const today = Date.parse('2026-09-07T00:00:00Z'); +const days = Array.from({ length: 28 }, (_, index) => new Date(today - (28 - index) * dayMs).toISOString().slice(0, 10)); +const dailySource = { kind: 'TrendsQuery', interval: 'day', series: [{ math: 'total' }] }; + +test('report fetches dashboard details when paginated list summaries omit tiles and excludes untracked dashboards', async () => { + const details: number[] = []; + const offsets: number[] = []; + const client = { async GET(path: string, options: any) { + if (path === '/dashboards/') { + const offset = options.params.query.offset ?? 0; + offsets.push(offset); + return { data: offset === 0 + ? { results: [{ id: 99, name: 'Legacy', tags: ['gtm'] }], next: 'next' } + : { results: [{ id: 10, name: 'Managed', tags: ['gtm'] }], next: null } }; + } + if (path === '/dashboards/{id}/') { + details.push(options.params.path.id); + return { data: { id: 10, name: 'Managed', tiles: [{ insight: { id: 7 } }] } }; + } + assert.equal(path, '/insights/{id}/'); + assert.equal(options.params.path.id, 7); + return { data: { id: 7, name: 'Accepted', query: { source: dailySource }, result: [{ days, data: Array(28).fill(1) }] } }; + } }; + const report = await generateReport({ client, asOf, dashboardIds: [10] }); + assert.deepEqual(offsets, [0, 1]); + assert.deepEqual(details, [10]); + assert.match(report.markdown, /\| Accepted \| 7 \| 7/); + assert.doesNotMatch(report.markdown, /Legacy/); +}); + +test('report refuses missing dashboard tile data instead of rendering an empty section', async () => { + const client = { async GET(path: string) { + return { data: path === '/dashboards/' + ? { results: [{ id: 10, name: 'Managed', tags: ['gtm'] }] } + : { id: 10, name: 'Managed' } }; + } }; + await assert.rejects(generateReport({ client, asOf, dashboardIds: [10] }), /tiles/i); +}); + +test('report identifies a dashboard with no insight tiles as unavailable', async () => { + const client = { async GET(path: string) { + return { data: path === '/dashboards/' + ? { results: [{ id: 10, name: 'Managed', tags: ['gtm'] }] } + : { id: 10, name: 'Managed', tiles: [] } }; + } }; + const report = await generateReport({ client, asOf, dashboardIds: [10] }); + assert.match(report.markdown, /Unavailable: no insight tiles/); +}); + +test('report prefers custom series names over identical raw event labels', () => { + const source = { ...dailySource, series: [{ math: 'total', custom_name: 'Dialog opened' }, { math: 'total', custom_name: 'Copy attempted' }] }; + const rows = insightReportRows({ id: 1, name: 'Intent', query: { source }, result: [{ label: 'marketing:cta_click', days, data: Array(28).fill(1) }, { label: 'marketing:cta_click', days, data: Array(28).fill(2) }] }, asOf); + assert.deepEqual(rows.map(row => row.metric), ['Intent — Dialog opened', 'Intent — Copy attempted']); +}); + +test('report keeps additive series separate and never sums unique daily actors into weekly users', () => { + const source = { kind: 'TrendsQuery', interval: 'day', series: [{ math: 'total' }, { math: 'total' }] }; + const rows = insightReportRows({ id: 1, name: 'Runtime', query: { kind: 'InsightVizNode', source }, result: [{ label: 'Starts', days, data: Array(28).fill(1) }, { label: 'Ends', days, data: Array(28).fill(2) }] }, asOf); + assert.equal(rows.length, 2); + assert.deepEqual(rows.map(row => row.thisWeek), [7, 14]); + const unique = insightReportRows({ id: 2, name: 'Users', query: { source: { ...source, series: [{ math: 'dau' }] } }, result: [{ data: Array(28).fill(1) }] }); + assert.equal(unique[0].thisWeek, null); + assert.match(unique[0].unavailable ?? '', /unique/i); +}); + +test('report uses dated completed UTC days, ignoring older data and the partial current day', () => { + // Reverse order deliberately: dates, rather than array position, define the window. + const resultDays = [...days, '2026-09-07', '2026-08-09'].reverse(); + const data = [...days.map((_, index) => index + 1), 10_000, 20_000].reverse(); + const rows = insightReportRows({ id: 1, name: 'Dated', query: { source: dailySource }, result: [{ days: resultDays, data }] }, asOf); + assert.deepEqual(rows[0].weeks, [28, 77, 126, 175]); + assert.equal(rows[0].thisWeek, 175); + assert.equal(rows[0].lastWeek, 126); +}); + +test('report rejects stale, undated, duplicate, gapped, partial and mismatched result dates', () => { + const invalidDays = [ + undefined, + days.map((day) => new Date(Date.parse(day) - dayMs).toISOString().slice(0, 10)), + [days[1], ...days.slice(1)], + [...days.slice(0, 10), ...days.slice(11), '2026-09-07'], + [...days.slice(0, -1), '2026-09-06T12:00:00Z'], + days.slice(1), + ]; + for (const resultDays of invalidDays) { + const rows = insightReportRows({ id: 1, name: 'Invalid', query: { source: dailySource }, result: [{ days: resultDays, data: Array(28).fill(1) }] }, asOf); + assert.equal(rows[0].thisWeek, null, JSON.stringify(resultDays)); + assert.match(rows[0].unavailable ?? '', /date|UTC|complete/i); + } +}); + +test('report never replaces a missing completed date with a zero', () => { + const rows = insightReportRows({ id: 1, name: 'Missing', query: { source: dailySource }, result: [{ days: days.slice(0, -1), data: Array(27).fill(0) }] }, asOf); + assert.equal(rows[0].thisWeek, null); + assert.deepEqual(rows[0].weeks, []); +}); + +test('report labels funnels and missing or incomplete results unavailable instead of zero', () => { + for (const insight of [ + { id: 1, name: 'Funnel', query: { source: { kind: 'FunnelsQuery' } }, result: [] }, + { id: 2, name: 'Missing', query: { source: { kind: 'TrendsQuery', interval: 'day', series: [{ math: 'total' }] } } }, + { id: 3, name: 'Short', query: { source: { kind: 'TrendsQuery', interval: 'day', series: [{ math: 'total' }] } }, result: [{ data: [1, 2] }] }, + ]) { + const rows = insightReportRows(insight); + assert.equal(rows[0].thisWeek, null); + assert.match(renderReport([{ name: 'GTM', rows }], '2026-09-07'), /Unavailable/); + } +}); test('sparkline: empty array returns dash', () => { assert.equal(sparkline([]), '—'); @@ -39,4 +150,6 @@ test('renderReport: produces stable markdown structure', () => { assert(out.includes('## Notes')); assert(out.includes('