From 6405e770987a2a1a234a9918d26e1c15180a5aee Mon Sep 17 00:00:00 2001 From: Brian Love Date: Sat, 5 Sep 2026 12:49:10 -0700 Subject: [PATCH] feat(lifecycle): add optional bounded Firecrawl company capture --- apps/lifecycle/README.md | 8 + apps/lifecycle/src/campaign/send.ts | 4 +- .../src/enrichment/company-capture.spec.ts | 117 +++++++ .../src/enrichment/company-capture.ts | 44 +++ .../lifecycle/src/enrichment/company-fetch.ts | 16 +- .../src/enrichment/firecrawl.spec.ts | 323 ++++++++++++++++++ apps/lifecycle/src/enrichment/firecrawl.ts | 277 +++++++++++++++ 7 files changed, 785 insertions(+), 4 deletions(-) create mode 100644 apps/lifecycle/src/enrichment/company-capture.spec.ts create mode 100644 apps/lifecycle/src/enrichment/company-capture.ts create mode 100644 apps/lifecycle/src/enrichment/firecrawl.spec.ts create mode 100644 apps/lifecycle/src/enrichment/firecrawl.ts diff --git a/apps/lifecycle/README.md b/apps/lifecycle/README.md index 437d5f523..242a374ec 100644 --- a/apps/lifecycle/README.md +++ b/apps/lifecycle/README.md @@ -26,3 +26,11 @@ The app's Vercel project must use `apps/lifecycle` as its root directory, enable Keep `LIFECYCLE_CRON_ENABLED` unset or set to anything other than the exact value `true` until the preview dogfood checklist in `docs/superpowers/runbooks/2026-08-31-growth-lifecycle-cutover.md` passes. In particular, verify outer auth on all Dawn surfaces, named-thread dispatch, duplicate invocation behavior, recovery pause/resume, cancellation/AbortSignal propagation, and Dawn persistence across fresh instances. Send findings to Dawn task `01a05e2f-7e93-7bd0-af74-f13d5a7719cd` for generalized backport. Use [DOGFOOD.md](./DOGFOOD.md) for the provider-free setup, probe, and exact cleanup commands. The harness binds the growth target to a database-owned comment sentinel and binds each authenticated lifecycle health response to Vercel's `VERCEL_DEPLOYMENT_ID`; it also validates lifecycle origins in memory before making requests. + +## Company evidence capture + +`LIFECYCLE_COMPANY_CAPTURE_PROVIDER` defaults to `direct`, preserving the existing company-page fetch. To explicitly enable managed homepage capture, set it to exactly `firecrawl` and configure the server-only `FIRECRAWL_API_KEY`. Configuration is checked only when enrichment needs company evidence; it does not gate email delivery. Invalid configuration and provider failures use the existing enrichment retry handling, without a direct-fetch fallback. + +Firecrawl capture makes one fresh homepage request with the basic proxy, a 10-second provider timeout, a 15-second total deadline, and a 2 MiB response limit. The existing HTML extractor produces the same bounded evidence schema. It accepts a changed final company hostname only when Firecrawl reports the requested source URL and a valid public HTTPS final URL. Local checks validate the input and final hostnames; Firecrawl owns remote DNS resolution and intermediate redirect safety. This is a provider trust boundary, not the direct fetcher's DNS-pinned transport. + +Capture logs contain provider, outcome, status, byte count, and reported credits where available (direct capture also identifies the fixed requested path). They exclude page text, company URLs, and credentials. Keep the default provider until an account key is configured and an authenticated capture is verified; public keyless experiments do not verify the production account integration. diff --git a/apps/lifecycle/src/campaign/send.ts b/apps/lifecycle/src/campaign/send.ts index 7f351cae4..6bfb24951 100644 --- a/apps/lifecycle/src/campaign/send.ts +++ b/apps/lifecycle/src/campaign/send.ts @@ -37,7 +37,7 @@ import { import { Resend } from 'resend'; import { generateEnrichmentArtifact } from '../enrichment/anthropic.js'; -import { fetchCompanyEvidence } from '../enrichment/company-fetch.js'; +import { createCompanyCapture } from '../enrichment/company-capture.js'; import { buildResearchInput } from '../enrichment/research-input.js'; import { EnrichmentArtifactSchema, @@ -886,7 +886,7 @@ export function createDefaultLifecycleJobDependencies( claimInternalNotification: claimInternalNotificationSubmission, markInternalNotificationUnknown, failJob: failLeasedJob, - fetchCompanyEvidence, + fetchCompanyEvidence: createCompanyCapture(environment), async readDeterministicScore(executor, contactId) { const score = await recomputeContactScore(executor, { contactId, diff --git a/apps/lifecycle/src/enrichment/company-capture.spec.ts b/apps/lifecycle/src/enrichment/company-capture.spec.ts new file mode 100644 index 000000000..f77c707d3 --- /dev/null +++ b/apps/lifecycle/src/enrichment/company-capture.spec.ts @@ -0,0 +1,117 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { direct, managed } = vi.hoisted(() => ({ + direct: vi.fn(), + managed: vi.fn(), +})); +vi.mock('./company-fetch.js', () => ({ fetchCompanyEvidence: direct })); +vi.mock('./firecrawl.js', () => ({ fetchFirecrawlCompanyEvidence: managed })); + +import { createCompanyCapture } from './company-capture.js'; + +beforeEach(() => { + direct.mockReset().mockResolvedValue([]); + managed.mockReset().mockResolvedValue([]); +}); + +describe('configured company capture', () => { + it('reports configuration failures without logging configuration values', async () => { + const log = vi.spyOn(console, 'info').mockImplementation(() => undefined); + try { + await expect( + createCompanyCapture({ + LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'secret-invalid', + })('example.com', new AbortController().signal) + ).rejects.toThrow('company_capture_invalid_provider'); + expect(log).toHaveBeenCalledWith('company_capture', { + outcome: 'invalid_provider', + }); + await expect( + createCompanyCapture({ + LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl', + })('example.com', new AbortController().signal) + ).rejects.toThrow('company_capture_missing_key'); + expect(log).toHaveBeenCalledWith('company_capture', { + provider: 'firecrawl', + outcome: 'missing_key', + }); + } finally { + log.mockRestore(); + } + }); + it.each([undefined, 'direct'])( + 'keeps %s on direct capture', + async (provider) => { + const signal = new AbortController().signal; + await createCompanyCapture({ + LIFECYCLE_COMPANY_CAPTURE_PROVIDER: provider, + })('example.com', signal); + expect(direct).toHaveBeenCalledWith('example.com', signal, { + onDiagnostic: expect.any(Function), + }); + expect(managed).not.toHaveBeenCalled(); + } + ); + + it('selects Firecrawl only with explicit configuration', async () => { + const signal = new AbortController().signal; + await createCompanyCapture({ + LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl', + FIRECRAWL_API_KEY: 'fixture-key', + })('example.com', signal); + expect(managed).toHaveBeenCalledWith('example.com', signal, { + apiKey: 'fixture-key', + onDiagnostic: expect.any(Function), + }); + expect(direct).not.toHaveBeenCalled(); + }); + + it('validates lazily and never falls back on invalid configuration', async () => { + const capture = createCompanyCapture({ + LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'invalid-secret-value', + }); + await expect( + capture('example.com', new AbortController().signal) + ).rejects.toThrow('company_capture_invalid_provider'); + expect(direct).not.toHaveBeenCalled(); + expect(managed).not.toHaveBeenCalled(); + }); + + it.each([undefined, '', ' '])( + 'requires a configured key before calling the provider: %s', + async (key) => { + const capture = createCompanyCapture({ + LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl', + FIRECRAWL_API_KEY: key, + }); + await expect( + capture('example.com', new AbortController().signal) + ).rejects.toThrow('company_capture_missing_key'); + expect(managed).not.toHaveBeenCalled(); + expect(direct).not.toHaveBeenCalled(); + } + ); + + it('preserves provider failures without a second capture attempt', async () => { + managed.mockRejectedValue(new Error('firecrawl_provider_error')); + await expect( + createCompanyCapture({ + LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl', + FIRECRAWL_API_KEY: 'fixture-key', + })('example.com', new AbortController().signal) + ).rejects.toThrow('firecrawl_provider_error'); + expect(managed).toHaveBeenCalledTimes(1); + expect(direct).not.toHaveBeenCalled(); + }); + + it('does not return evidence when cancellation arrives during capture', async () => { + const controller = new AbortController(); + direct.mockImplementation(async () => { + controller.abort(new Error('cancelled')); + return []; + }); + await expect( + createCompanyCapture({})('example.com', controller.signal) + ).rejects.toThrow('cancelled'); + }); +}); diff --git a/apps/lifecycle/src/enrichment/company-capture.ts b/apps/lifecycle/src/enrichment/company-capture.ts new file mode 100644 index 000000000..d35495f4b --- /dev/null +++ b/apps/lifecycle/src/enrichment/company-capture.ts @@ -0,0 +1,44 @@ +import { fetchCompanyEvidence } from './company-fetch.js'; +import { fetchFirecrawlCompanyEvidence } from './firecrawl.js'; +import type { CompanyPageEvidence } from './schema.js'; + +function report(diagnostic: object): void { + try { + console.info('company_capture', diagnostic); + } catch { + // Observability must not change capture behavior. + } +} + +export function createCompanyCapture( + environment: Record +): (domain: string, signal: AbortSignal) => Promise { + return async (domain, signal) => { + signal.throwIfAborted(); + const provider = environment['LIFECYCLE_COMPANY_CAPTURE_PROVIDER']; + let evidence: CompanyPageEvidence[]; + if (provider === undefined || provider === 'direct') { + evidence = await fetchCompanyEvidence(domain, signal, { + onDiagnostic: (diagnostic) => + report({ provider: 'direct', ...diagnostic }), + }); + } else if (provider === 'firecrawl') { + const apiKey = environment['FIRECRAWL_API_KEY']?.trim(); + if (!apiKey) { + report({ provider: 'firecrawl', outcome: 'missing_key' }); + signal.throwIfAborted(); + throw new Error('company_capture_missing_key'); + } + evidence = await fetchFirecrawlCompanyEvidence(domain, signal, { + apiKey, + onDiagnostic: report, + }); + } else { + report({ outcome: 'invalid_provider' }); + signal.throwIfAborted(); + throw new Error('company_capture_invalid_provider'); + } + signal.throwIfAborted(); + return evidence; + }; +} diff --git a/apps/lifecycle/src/enrichment/company-fetch.ts b/apps/lifecycle/src/enrichment/company-fetch.ts index 363cfb5a4..6d6fe6890 100644 --- a/apps/lifecycle/src/enrichment/company-fetch.ts +++ b/apps/lifecycle/src/enrichment/company-fetch.ts @@ -381,7 +381,7 @@ function isPublicAddress(address: string): boolean { async function resolvePublicAddresses( hostname: string, signal: AbortSignal, - dependencies: CompanyFetchDependencies + dependencies: Pick ): Promise { const addresses = await dependencies.resolve(hostname, signal); signal.throwIfAborted(); @@ -396,6 +396,18 @@ async function resolvePublicAddresses( return addresses; } +/** Reuse the direct fetcher's hostname and public-address policy for providers. */ +export async function validatePublicCompanyHostname( + domain: string, + signal: AbortSignal, + resolve: CompanyFetchDependencies['resolve'] = resolveWithNodeDns +): Promise { + const hostname = validatedCompanyHostname(domain); + signal.throwIfAborted(); + await resolvePublicAddresses(hostname, signal, { resolve }); + return hostname; +} + function validatedRedirectUrl( location: string, current: URL, @@ -572,7 +584,7 @@ function descriptionValues( return values; } -function extractEvidence( +export function extractEvidence( body: Uint8Array ): Pick { const html = new TextDecoder('utf-8', { fatal: false }).decode(body); diff --git a/apps/lifecycle/src/enrichment/firecrawl.spec.ts b/apps/lifecycle/src/enrichment/firecrawl.spec.ts new file mode 100644 index 000000000..70995a554 --- /dev/null +++ b/apps/lifecycle/src/enrichment/firecrawl.spec.ts @@ -0,0 +1,323 @@ +import { createHash } from 'node:crypto'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + fetchFirecrawlCompanyEvidence, + type FirecrawlOptions, +} from './firecrawl.js'; + +const html = + 'Example

Tools

Useful tools

'; +const metadata = { + sourceURL: 'https://example.com/', + url: 'https://www.example.com/', + statusCode: 200, +}; +const payload = (data = {}) => ({ + success: true, + data: { html, metadata, ...data }, +}); +function setup(body: unknown = payload()) { + const fetch = vi + .fn() + .mockResolvedValue(Response.json(body)); + const resolve = vi.fn().mockResolvedValue(['93.184.216.34']); + const options: FirecrawlOptions = { + apiKey: 'test-key', + fetch, + resolve, + now: () => new Date('2026-09-01T12:00:00Z'), + }; + return { fetch, resolve, options }; +} +const run = ( + options: FirecrawlOptions, + domain = 'example.com', + signal = new AbortController().signal +) => fetchFirecrawlCompanyEvidence(domain, signal, options); +afterEach(() => vi.useRealTimers()); +describe('Firecrawl homepage evidence', () => { + it('uses exactly one fixed authenticated scrape and maps extracted HTML with actual URL provenance', async () => { + const { fetch, options } = setup(); + const result = await run(options); + expect(result).toEqual([ + { + canonicalUrl: metadata.url, + retrievedAt: '2026-09-01T12:00:00.000Z', + contentHash: createHash('sha256').update(html).digest('hex'), + facts: ['Example', 'Tools'], + snippets: ['Useful tools'], + }, + ]); + expect(fetch).toHaveBeenCalledTimes(1); + const [url, init] = fetch.mock.calls[0]; + expect(url).toBe('https://api.firecrawl.dev/v2/scrape'); + expect(init).toMatchObject({ + method: 'POST', + redirect: 'error', + headers: { authorization: 'Bearer test-key' }, + }); + expect(JSON.parse(String(init?.body))).toEqual({ + url: 'https://example.com/', + formats: ['html'], + onlyMainContent: true, + maxAge: 0, + timeout: 10000, + proxy: 'basic', + }); + }); + it('requires the key before DNS or network', async () => { + const { options, fetch, resolve } = setup(); + await expect(run({ ...options, apiKey: ' ' })).rejects.toThrow( + 'configuration' + ); + expect(fetch).not.toHaveBeenCalled(); + expect(resolve).not.toHaveBeenCalled(); + }); + it.each([ + '127.0.0.1', + 'localhost', + 'https://example.com', + 'example.com/path', + ])('rejects unsafe input %s before network', async (domain) => { + const { options, fetch } = setup(); + await expect(run(options, domain)).rejects.toThrow('security_rejected'); + expect(fetch).not.toHaveBeenCalled(); + }); + it('rejects mixed public/private DNS answers', async () => { + const { options, resolve, fetch } = setup(); + resolve.mockResolvedValue(['93.184.216.34', '10.0.0.1']); + await expect(run(options)).rejects.toThrow('security_rejected'); + expect(fetch).not.toHaveBeenCalled(); + }); + it.each([ + { ...metadata, sourceURL: 'https://unrelated.com/' }, + { sourceURL: metadata.sourceURL, ogUrl: metadata.url, statusCode: 200 }, + { ...metadata, url: 'http://example.com/' }, + { ...metadata, url: 'https://127.0.0.1/' }, + { ...metadata, url: 'https://example.com/?token=secret' }, + { ...metadata, url: 'https://example.com/#fragment' }, + { ...metadata, url: 'https://user:pass@example.com/' }, + { ...metadata, url: 'https://example.com:444/' }, + { ...metadata, url: `https://example.com/${'x'.repeat(500)}` }, + ])('rejects invalid provenance %#', async (invalid) => { + await expect( + run(setup(payload({ metadata: invalid })).options) + ).rejects.toThrow(); + }); + it('validates final hostname DNS', async () => { + const { options, resolve } = setup(); + resolve.mockImplementation(async (host) => + host === 'www.example.com' ? ['10.0.0.1'] : ['93.184.216.34'] + ); + await expect(run(options)).rejects.toThrow('security_rejected'); + }); + it.each([ + payload({ html: '' }), + payload({ metadata: { ...metadata, statusCode: 404 } }), + payload({ html: '' }), + ])('returns no evidence for empty or missing pages %#', async (body) => { + await expect(run(setup(body).options)).resolves.toEqual([]); + }); + it.each([401, 402, 429, 500, 302])( + 'rejects API HTTP %s without logging body or retrying', + async (status) => { + const { options, fetch } = setup(); + fetch.mockResolvedValue(new Response('private body', { status })); + const diagnostics: unknown[] = []; + await expect( + run({ ...options, onDiagnostic: (d) => diagnostics.push(d) }) + ).rejects.toThrow('api_http_error'); + expect(fetch).toHaveBeenCalledTimes(1); + expect(diagnostics).toEqual([ + { provider: 'firecrawl', outcome: 'api_http_error', apiStatus: status }, + ]); + } + ); + it.each([ + { success: false, error: 'private body' }, + { success: true, data: null }, + { + success: true, + data: { html, metadata: { ...metadata, statusCode: 503 } }, + }, + ])('rejects provider failures %#', async (body) => { + await expect(run(setup(body).options)).rejects.toThrow(); + }); + it.each([true, false])( + 'caps advertised and streamed JSON bytes (%s)', + async (advertised) => { + const { options, fetch } = setup(); + fetch.mockResolvedValue( + new Response('x'.repeat(2 * 1024 * 1024 + 1), { + headers: advertised + ? { 'content-length': String(2 * 1024 * 1024 + 1) } + : {}, + }) + ); + await expect(run(options)).rejects.toThrow('response_too_large'); + } + ); + it.each(['dns', 'fetch', 'body'])( + 'bounds stalled %s at total 15 seconds', + async (stage) => { + vi.useFakeTimers(); + const { options, fetch, resolve } = setup(); + if (stage === 'dns') + resolve.mockImplementation( + () => + new Promise(() => { + /* Simulate a stalled transport. */ + }) + ); + if (stage === 'fetch') + fetch.mockImplementation( + () => + new Promise(() => { + /* Simulate a stalled transport. */ + }) + ); + if (stage === 'body') + fetch.mockResolvedValue( + new Response( + new ReadableStream({ + pull: () => + new Promise(() => { + /* Simulate a stalled transport. */ + }), + cancel: () => + new Promise(() => { + /* Simulate a stalled transport. */ + }), + }) + ) + ); + const checked = expect(run(options)).rejects.toThrow('timeout'); + await vi.advanceTimersByTimeAsync(15000); + await checked; + } + ); + it('propagates cancellation without late evidence even if callback aborts', async () => { + const { options } = setup(); + const controller = new AbortController(); + const reason = new Error('caller'); + await expect( + run( + { ...options, onDiagnostic: () => controller.abort(reason) }, + 'example.com', + controller.signal + ) + ).rejects.toBe(reason); + }); + it.each(['dns', 'fetch', 'body'])( + 'propagates caller cancellation during %s without waiting for transport', + async (stage) => { + const { options, fetch, resolve } = setup(); + const controller = new AbortController(); + const reason = new Error('caller'); + if (stage === 'dns') + resolve.mockImplementation( + () => + new Promise(() => { + /* Simulate a stalled transport. */ + }) + ); + if (stage === 'fetch') + fetch.mockImplementation( + () => + new Promise(() => { + /* Simulate a stalled transport. */ + }) + ); + if (stage === 'body') + fetch.mockResolvedValue( + new Response( + new ReadableStream({ + pull: () => + new Promise(() => { + /* Simulate a stalled transport. */ + }), + }) + ) + ); + const pending = run(options, 'example.com', controller.signal); + const checked = expect(pending).rejects.toBe(reason); + await new Promise((resolve) => setTimeout(resolve, 0)); + controller.abort(reason); + await checked; + } + ); + it('does not request a pre-cancelled capture', async () => { + const { options, fetch, resolve } = setup(); + const controller = new AbortController(); + controller.abort(); + await expect( + run(options, 'example.com', controller.signal) + ).rejects.toHaveProperty('name', 'AbortError'); + expect(fetch).not.toHaveBeenCalled(); + expect(resolve).not.toHaveBeenCalled(); + }); + it('checks cancellation after the clock callback', async () => { + const { options } = setup(); + const controller = new AbortController(); + const reason = new Error('caller'); + await expect( + run( + { + ...options, + now: () => { + controller.abort(reason); + return new Date(); + }, + }, + 'example.com', + controller.signal + ) + ).rejects.toBe(reason); + }); + it('keeps the existing six-item and 240-character extraction bounds', async () => { + const body = `${'x'.repeat(300)}${'

Heading

'.repeat( + 10 + )}${Array.from( + { length: 10 }, + (_, i) => `

${i}${'p'.repeat(300)}

` + ).join('')}`; + const result = await run(setup(payload({ html: body })).options); + expect(result[0].facts.length).toBeLessThanOrEqual(6); + expect(result[0].snippets).toHaveLength(6); + expect( + [...result[0].facts, ...result[0].snippets].every( + (text) => text.length <= 240 + ) + ).toBe(true); + }); + it('rejects malformed JSON without leaking its body', async () => { + const { options, fetch } = setup(); + fetch.mockResolvedValue(new Response('private invalid json')); + await expect(run(options)).rejects.toThrow(/^invalid_response$/); + }); + it('ignores observer errors and exposes only bounded diagnostic fields', async () => { + const { options } = setup( + payload({ metadata: { ...metadata, creditsUsed: 1, private: 'secret' } }) + ); + const diagnostics: unknown[] = []; + expect( + await run({ + ...options, + onDiagnostic: (d) => { + diagnostics.push(d); + throw new Error('observer'); + }, + }) + ).toHaveLength(1); + expect(diagnostics).toEqual([ + { + provider: 'firecrawl', + outcome: 'captured', + apiStatus: 200, + pageStatus: 200, + bytes: expect.any(Number), + credits: 1, + }, + ]); + }); +}); diff --git a/apps/lifecycle/src/enrichment/firecrawl.ts b/apps/lifecycle/src/enrichment/firecrawl.ts new file mode 100644 index 000000000..3117202b5 --- /dev/null +++ b/apps/lifecycle/src/enrichment/firecrawl.ts @@ -0,0 +1,277 @@ +import { createHash } from 'node:crypto'; +import { + CompanyFetchSecurityError, + extractEvidence, + validatePublicCompanyHostname, + type CompanyFetchDependencies, +} from './company-fetch.js'; +import { + CompanyPageEvidenceSchema, + type CompanyPageEvidence, +} from './schema.js'; + +const MAX_BYTES = 2 * 1024 * 1024; +type Failure = + | 'configuration' + | 'security_rejected' + | 'invalid_response' + | 'invalid_provenance' + | 'api_http_error' + | 'page_http_error' + | 'response_too_large' + | 'timeout' + | 'transport_failure'; +export interface FirecrawlDiagnostic { + provider: 'firecrawl'; + outcome: 'captured' | 'no_evidence' | Failure; + apiStatus?: number; + pageStatus?: number; + bytes?: number; + credits?: number; +} +export interface FirecrawlOptions { + apiKey: string; + fetch?: typeof fetch; + resolve?: CompanyFetchDependencies['resolve']; + now?: () => Date; + onDiagnostic?: (diagnostic: FirecrawlDiagnostic) => void; +} +class FirecrawlError extends Error { + constructor(readonly code: Failure) { + super(code); + this.name = 'FirecrawlError'; + } +} + +// Race every asynchronous stage, including injected transports and stalled bodies. +function abortable(promise: Promise, signal: AbortSignal): Promise { + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason); + signal.addEventListener('abort', abort, { once: true }); + promise + .then(resolve, reject) + .finally(() => signal.removeEventListener('abort', abort)); + }); +} + +function safeUrl(value: unknown): URL { + if (typeof value !== 'string' || value.length > 500 || value !== value.trim()) + throw new FirecrawlError('invalid_provenance'); + let url: URL; + try { + url = new URL(value); + } catch { + throw new FirecrawlError('invalid_provenance'); + } + if ( + url.protocol !== 'https:' || + url.username || + url.password || + url.port || + url.search || + url.hash || + value.includes('?') || + value.includes('#') + ) + throw new FirecrawlError('invalid_provenance'); + return url; +} +function record(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new FirecrawlError('invalid_response'); + return value as Record; +} +function dispose(body: ReadableStream | null): void { + if (body) + void body.cancel().catch(() => { + /* Best-effort disposal. */ + }); +} +async function boundedJson( + response: Response, + signal: AbortSignal, + diagnostic: FirecrawlDiagnostic +): Promise { + const length = Number(response.headers.get('content-length')); + if (Number.isFinite(length) && length > MAX_BYTES) { + diagnostic.bytes = Math.min(length, Number.MAX_SAFE_INTEGER); + dispose(response.body); + throw new FirecrawlError('response_too_large'); + } + if (!response.body) throw new FirecrawlError('invalid_response'); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let bytes = 0; + try { + while (true) { + const { done, value } = await abortable(reader.read(), signal); + signal.throwIfAborted(); + if (done) break; + bytes += value.byteLength; + diagnostic.bytes = bytes; + if (bytes > MAX_BYTES) throw new FirecrawlError('response_too_large'); + chunks.push(value); + } + } catch (error) { + // Cleanup must not extend the deadline if cancellation itself stalls. + void reader.cancel().catch(() => { + /* Preserve the read failure. */ + }); + throw error; + } finally { + reader.releaseLock(); + } + const body = Buffer.concat(chunks, bytes); + try { + return JSON.parse(body.toString('utf8')) as unknown; + } catch { + throw new FirecrawlError('invalid_response'); + } +} + +/** One fresh homepage scrape; errors are code-only so the campaign can retry safely. */ +export async function fetchFirecrawlCompanyEvidence( + domain: string, + signal: AbortSignal, + options: FirecrawlOptions +): Promise { + const deadline = new AbortController(); + const timer = setTimeout( + () => deadline.abort(new FirecrawlError('timeout')), + 15_000 + ); + const combined = AbortSignal.any([signal, deadline.signal]); + const diagnostic: FirecrawlDiagnostic = { + provider: 'firecrawl', + outcome: 'transport_failure', + }; + const report = () => { + try { + options.onDiagnostic?.({ ...diagnostic }); + } catch { + /* Observational only. */ + } + }; + try { + combined.throwIfAborted(); + if (!options.apiKey?.trim()) throw new FirecrawlError('configuration'); + const hostname = await abortable( + validatePublicCompanyHostname(domain, combined, options.resolve), + combined + ); + combined.throwIfAborted(); + const requestedUrl = `https://${hostname}/`; + const pending = (options.fetch ?? fetch)( + 'https://api.firecrawl.dev/v2/scrape', + { + method: 'POST', + redirect: 'error', + signal: combined, + headers: { + authorization: `Bearer ${options.apiKey}`, + 'content-type': 'application/json', + accept: 'application/json', + }, + body: JSON.stringify({ + url: requestedUrl, + formats: ['html'], + onlyMainContent: true, + maxAge: 0, + timeout: 10000, + proxy: 'basic', + }), + } + ); + void pending.then( + (response) => { + if (combined.aborted) dispose(response.body); + }, + () => { + /* The awaited request below handles transport errors. */ + } + ); + const response = await abortable(pending, combined); + combined.throwIfAborted(); + diagnostic.apiStatus = response.status; + if (!response.ok) { + dispose(response.body); + throw new FirecrawlError('api_http_error'); + } + const result = record(await boundedJson(response, combined, diagnostic)); + combined.throwIfAborted(); + if (result.success !== true) throw new FirecrawlError('invalid_response'); + const data = record(result.data); + const metadata = record(data.metadata); + const source = safeUrl(metadata.sourceURL); + const final = safeUrl(metadata.url); + if (source.toString() !== requestedUrl) + throw new FirecrawlError('invalid_provenance'); + await abortable( + validatePublicCompanyHostname(final.hostname, combined, options.resolve), + combined + ); + combined.throwIfAborted(); + const status = metadata.statusCode; + if ( + typeof status !== 'number' || + !Number.isInteger(status) || + status < 100 || + status > 599 + ) + throw new FirecrawlError('invalid_response'); + diagnostic.pageStatus = status; + if ( + typeof metadata.creditsUsed === 'number' && + Number.isSafeInteger(metadata.creditsUsed) && + metadata.creditsUsed >= 0 + ) + diagnostic.credits = metadata.creditsUsed; + if (status !== 404 && (status < 200 || status >= 300)) + throw new FirecrawlError('page_http_error'); + if (status === 404) { + diagnostic.outcome = 'no_evidence'; + report(); + combined.throwIfAborted(); + return []; + } + if (typeof data.html !== 'string') + throw new FirecrawlError('invalid_response'); + const body = Buffer.from(data.html, 'utf8'); + const extracted = extractEvidence(body); + if (!extracted.facts.length && !extracted.snippets.length) { + diagnostic.outcome = 'no_evidence'; + report(); + combined.throwIfAborted(); + return []; + } + const evidence = CompanyPageEvidenceSchema.parse({ + canonicalUrl: final.toString(), + retrievedAt: (options.now ?? (() => new Date()))().toISOString(), + contentHash: createHash('sha256').update(body).digest('hex'), + ...extracted, + }); + combined.throwIfAborted(); + diagnostic.outcome = 'captured'; + report(); + combined.throwIfAborted(); + return [evidence]; + } catch (error) { + signal.throwIfAborted(); + const failure = deadline.signal.aborted + ? new FirecrawlError('timeout') + : error instanceof FirecrawlError + ? error + : new FirecrawlError( + error instanceof CompanyFetchSecurityError + ? 'security_rejected' + : 'transport_failure' + ); + diagnostic.outcome = failure.code; + report(); + signal.throwIfAborted(); + throw failure; + } finally { + clearTimeout(timer); + } +}