diff --git a/apps/growth-research/README.md b/apps/growth-research/README.md index 9596ad371..0d2181693 100644 --- a/apps/growth-research/README.md +++ b/apps/growth-research/README.md @@ -17,9 +17,19 @@ npx tsx apps/growth-research/scripts/research-pilot.mts acquire --output /absolu ``` These commands return UUIDs for immutable JSON files in the selected output directory. -Acquisition records include complete, partial, empty and failed outcomes. The existing -fetcher can skip unusable pages, so missing paths have an unknown reason; redirects may -make the original path indeterminate. Review the captured corpus before model calls: +Acquisition records include complete, partial, empty and failed outcomes. Each capture's +`pageDiagnostics` records the original requested path, a bounded outcome code, HTTP +status and known byte count when available. Outcomes distinguish capture, access denial +(403), rate limiting (429), other HTTP failures, oversized pages, request timeout, +transport failure, rejected redirects, missing redirect locations, exhausted redirect +budget and security rejection. Diagnostics emitted before a security rejection remain +in the failed capture; caller cancellation still rejects acquisition. Diagnostics contain +no response bodies, exception messages or redirect URLs. `access_denied` records HTTP +403; it does not prove bot detection. Missing diagnostic entries can mean a page was +not attempted or an older injected capture function did not support diagnostics. The +250 KiB page limit, five-second timeout, three-total-redirect budget, exact-host redirect +policy and SSRF controls are unchanged. The existing unavailable-path summary uses final URLs and can +remain indeterminate after redirects. Review the captured corpus before model calls: remove personal biography/contact snippets, retain empty cases and failures, and fill expected claims/unknowns from the actual captured evidence. Save the reviewed corpus under a new name/version. Acquisition is preparation, not a human quality label. diff --git a/apps/growth-research/src/pilot/acquisition.ts b/apps/growth-research/src/pilot/acquisition.ts index ed4b4e18e..bfeff2339 100644 --- a/apps/growth-research/src/pilot/acquisition.ts +++ b/apps/growth-research/src/pilot/acquisition.ts @@ -1,4 +1,8 @@ -import { fetchCompanyEvidence } from '../../../lifecycle/src/enrichment/company-fetch.js'; +import { + fetchCompanyEvidence, + type CompanyFetchOverrides, + type CompanyPageDiagnostic, +} from '../../../lifecycle/src/enrichment/company-fetch.js'; import type { CompanyPageEvidence } from '../../../lifecycle/src/enrichment/schema.js'; const expectedPaths = ['/', '/about', '/pricing']; @@ -7,7 +11,8 @@ export async function acquireCompanies( signal: AbortSignal, capture: ( domain: string, - signal: AbortSignal + signal: AbortSignal, + options?: Pick ) => Promise = fetchCompanyEvidence ) { if ( @@ -36,14 +41,18 @@ export async function acquireCompanies( reason: 'unavailable' | 'capture_failed' | null; redirectedPathsIndeterminate: boolean; filteredIdentityItems: number; + pageDiagnostics: CompanyPageDiagnostic[]; }[] = []; for (const [index, domain] of domains.entries()) { signal.throwIfAborted(); const id = `public-${index + 1}`; let pages: CompanyPageEvidence[] = [], failed = false; + const pageDiagnostics: CompanyPageDiagnostic[] = []; try { - pages = await capture(domain, signal); + pages = await capture(domain, signal, { + onDiagnostic: (diagnostic) => pageDiagnostics.push(diagnostic), + }); } catch { signal.throwIfAborted(); failed = true; @@ -92,6 +101,7 @@ export async function acquireCompanies( (path) => !expectedPaths.includes(path) ), filteredIdentityItems, + pageDiagnostics, }); } return { diff --git a/apps/growth-research/test/pilot-acquisition.spec.ts b/apps/growth-research/test/pilot-acquisition.spec.ts index e4b931c30..80674a21f 100644 --- a/apps/growth-research/test/pilot-acquisition.spec.ts +++ b/apps/growth-research/test/pilot-acquisition.spec.ts @@ -1,5 +1,34 @@ import { expect, it } from 'vitest'; import { acquireCompanies } from '../src/pilot/acquisition.js'; +// Exercise the same internal capture dependency used by pilot acquisition. +// eslint-disable-next-line @nx/enforce-module-boundaries +import { fetchCompanyEvidence } from '../../lifecycle/src/enrichment/company-fetch.js'; + +it('retains partial diagnostics when a later page rejects for security', async () => { + const result = await acquireCompanies( + ['atlas.example'], + new AbortController().signal, + (domain, signal, options) => + fetchCompanyEvidence(domain, signal, { + ...options, + resolve: async () => ['93.184.216.34'], + fetch: async (url) => + url.pathname === '/' + ? new Response('Atlas') + : new Response(null, { + status: 302, + headers: { location: 'https://unsafe.example/?secret=private' }, + }), + }) + ); + expect(result.captures[0].status).toBe('failed'); + expect(result.cases[0].pages).toEqual([]); + expect(result.captures[0].pageDiagnostics).toEqual([ + { requestedPath: '/', outcome: 'captured', status: 200, bytes: 20 }, + { requestedPath: '/about', outcome: 'redirect_rejected', status: 302 }, + ]); + expect(JSON.stringify(result)).not.toContain('private'); +}); it('keeps partial, empty, and failed company captures visible', async () => { const result = await acquireCompanies( diff --git a/apps/lifecycle/src/enrichment/company-fetch.spec.ts b/apps/lifecycle/src/enrichment/company-fetch.spec.ts index a834cf3ad..5b4e26f89 100644 --- a/apps/lifecycle/src/enrichment/company-fetch.spec.ts +++ b/apps/lifecycle/src/enrichment/company-fetch.spec.ts @@ -9,11 +9,157 @@ import { fetchCompanyEvidence, resolveWithNodeDns, type CompanyFetchDependencies, + type CompanyPageDiagnostic, type CompanyRequestInit, } from './company-fetch.js'; const NOW = new Date('2026-09-01T12:00:00.000Z'); +describe('page diagnostics', () => { + it.each([ + [403, 'access_denied'], + [429, 'rate_limited'], + [503, 'http_error'], + ] as const)( + 'reports HTTP %s without response content', + async (status, outcome) => { + const diagnostics: CompanyPageDiagnostic[] = []; + await expect( + fetchCompanyEvidence('example.com', new AbortController().signal, { + ...dependencies({ + fetch: async () => new Response('private body', { status }), + }), + onDiagnostic: (diagnostic) => diagnostics.push(diagnostic), + }) + ).resolves.toEqual([]); + expect(diagnostics).toEqual( + ['/', '/about', '/pricing'].map((requestedPath) => ({ + requestedPath, + outcome, + status, + })) + ); + } + ); + + it('records requested paths for redirected captures and isolates observer exceptions', async () => { + const diagnostics: CompanyPageDiagnostic[] = []; + const pages = await fetchCompanyEvidence( + 'example.com', + new AbortController().signal, + { + ...dependencies({ + fetch: async (url) => + url.pathname === '/about' + ? new Response(null, { + status: 302, + headers: { location: '/company?token=private' }, + }) + : okPage(), + }), + onDiagnostic: (diagnostic) => { + diagnostics.push(diagnostic); + throw new Error('observer'); + }, + } + ); + expect(pages).toHaveLength(3); + expect(diagnostics.map((d) => d.requestedPath)).toEqual([ + '/', + '/about', + '/pricing', + ]); + expect( + diagnostics.every( + (d) => + d.outcome === 'captured' && d.status === 200 && (d.bytes ?? 0) > 0 + ) + ).toBe(true); + expect(JSON.stringify(diagnostics)).not.toContain('private'); + }); + + it.each([ + 'missing_location', + 'redirect_limit', + 'redirect_rejected', + 'security_rejected', + 'page_too_large', + 'transport_failure', + 'timeout', + ] as const)('classifies %s without weakening policy', async (outcome) => { + const diagnostics: CompanyPageDiagnostic[] = []; + const ownTimeout = AbortSignal.abort(new Error('private timeout')); + const operation = fetchCompanyEvidence( + 'example.com', + new AbortController().signal, + { + ...dependencies({ + ...(outcome === 'timeout' + ? { + createTimeoutSignal: () => ({ + signal: ownTimeout, + clear: () => undefined, + }), + } + : {}), + resolve: async () => [ + outcome === 'security_rejected' ? '127.0.0.1' : '93.184.216.34', + ], + fetch: async () => { + if (outcome === 'transport_failure') + throw new Error('private transport'); + if (outcome === 'page_too_large') + return new Response('x', { + headers: { 'content-length': '256001' }, + }); + return new Response(null, { + status: 302, + headers: + outcome === 'missing_location' + ? {} + : { + location: + outcome === 'redirect_rejected' + ? 'https://other.example/?secret=private' + : '/loop', + }, + }); + }, + }), + onDiagnostic: (diagnostic) => { + diagnostics.push(diagnostic); + throw new Error('observer'); + }, + } + ); + if (outcome === 'security_rejected' || outcome === 'redirect_rejected') + await expect(operation).rejects.toThrow(/unsafe/iu); + else await expect(operation).resolves.toEqual([]); + expect(diagnostics[0]).toMatchObject({ requestedPath: '/', outcome }); + expect(JSON.stringify(diagnostics)).not.toContain('private'); + }); + + it('does not classify caller cancellation as a timeout or let an observer mask it', async () => { + const controller = new AbortController(); + const reason = new Error('caller stopped'); + const observer = vi.fn(() => { + throw new Error('observer'); + }); + await expect( + fetchCompanyEvidence('example.com', controller.signal, { + ...dependencies({ + fetch: async () => { + controller.abort(reason); + throw reason; + }, + }), + onDiagnostic: observer, + }) + ).rejects.toBe(reason); + expect(observer).not.toHaveBeenCalled(); + }); +}); + function dependencies( overrides: Partial = {} ): CompanyFetchDependencies { diff --git a/apps/lifecycle/src/enrichment/company-fetch.ts b/apps/lifecycle/src/enrichment/company-fetch.ts index d90b29dcb..363cfb5a4 100644 --- a/apps/lifecycle/src/enrichment/company-fetch.ts +++ b/apps/lifecycle/src/enrichment/company-fetch.ts @@ -60,6 +60,33 @@ export type HttpsRequestFactory = ( export interface CompanyFetchOverrides extends Partial { request?: HttpsRequestFactory; + /** Observational only: observer exceptions never affect capture. */ + onDiagnostic?: (diagnostic: CompanyPageDiagnostic) => void; +} + +export interface CompanyPageDiagnostic { + requestedPath: (typeof PAGE_PATHS)[number]; + outcome: + | 'captured' + | 'access_denied' + | 'rate_limited' + | 'http_error' + | 'page_too_large' + | 'timeout' + | 'transport_failure' + | 'redirect_rejected' + | 'missing_location' + | 'redirect_limit' + | 'security_rejected'; + status?: number; + /** Known body bytes read, or advertised bytes when rejected before reading. */ + bytes?: number; +} + +class CompanyPageTooLargeError extends Error { + constructor(readonly bytes: number) { + super('Company page exceeds 250 KiB'); + } } function defaultTimeoutSignal( @@ -411,7 +438,7 @@ async function readBoundedBody(response: Response): Promise { Number.parseInt(advertisedLength, 10) > MAX_PAGE_BYTES ) { await cancelResponseBody(response); - throw new Error('Company page exceeds 250 KiB'); + throw new CompanyPageTooLargeError(Number.parseInt(advertisedLength, 10)); } if (!response.body) return new Uint8Array(); @@ -424,7 +451,7 @@ async function readBoundedBody(response: Response): Promise { if (done) break; totalBytes += value.byteLength; if (totalBytes > MAX_PAGE_BYTES) { - throw new Error('Company page exceeds 250 KiB'); + throw new CompanyPageTooLargeError(totalBytes); } chunks.push(value); } @@ -572,6 +599,25 @@ export async function fetchCompanyEvidence( for (const path of PAGE_PATHS) { let currentUrl = new URL(path, `https://${hostname}/`); + let status: number | undefined; + let outcome: CompanyPageDiagnostic['outcome'] | undefined; + const report = ( + result: CompanyPageDiagnostic['outcome'], + bytes?: number + ) => { + try { + overrides.onDiagnostic?.({ + requestedPath: path, + outcome: result, + ...(status === undefined ? {} : { status }), + ...(bytes === undefined || !Number.isSafeInteger(bytes) + ? {} + : { bytes }), + }); + } catch { + // Observers cannot alter evidence, security rejection, or cancellation. + } + }; const timeout = dependencies.createTimeoutSignal( signal, REQUEST_TIMEOUT_MS @@ -594,19 +640,35 @@ export async function fetchCompanyEvidence( 'user-agent': 'ThreadplaneCompanyResearch/1.0', }, }); + status = response.status; if (REDIRECT_STATUSES.has(response.status)) { const location = response.headers.get('location'); await cancelResponseBody(response); - if (!location) + if (!location) { + outcome = 'missing_location'; throw new Error('Company redirect is missing Location'); + } redirects += 1; if (redirects > MAX_REDIRECTS) { + outcome = 'redirect_limit'; throw new Error('Company redirect limit exceeded'); } - currentUrl = validatedRedirectUrl(location, currentUrl, hostname); + try { + currentUrl = validatedRedirectUrl(location, currentUrl, hostname); + } catch (error) { + outcome = 'redirect_rejected'; + throw error; + } + status = undefined; continue; } if (!response.ok) { + outcome = + response.status === 403 + ? 'access_denied' + : response.status === 429 + ? 'rate_limited' + : 'http_error'; await cancelResponseBody(response); throw new Error(`Company page returned HTTP ${response.status}`); } @@ -619,6 +681,7 @@ export async function fetchCompanyEvidence( ...extractEvidence(body), }) ); + report('captured', body.byteLength); break; } } catch (error) { @@ -626,6 +689,17 @@ export async function fetchCompanyEvidence( // skipped so the remaining pages still yield evidence. The caller's // own abort and any SSRF violation propagate. signal.throwIfAborted(); + report( + outcome ?? + (error instanceof CompanyFetchSecurityError + ? 'security_rejected' + : error instanceof CompanyPageTooLargeError + ? 'page_too_large' + : timeout.signal.aborted + ? 'timeout' + : 'transport_failure'), + error instanceof CompanyPageTooLargeError ? error.bytes : undefined + ); if (error instanceof CompanyFetchSecurityError) throw error; } finally { timeout.clear();