feat(lifecycle): integrate self-hosted Firecrawl browser scraper

apps/lifecycle/README.md

@@ -29,8 +29,8 @@ Use [DOGFOOD.md](./DOGFOOD.md) for the provider-free setup, probe, and exact cle
 
 ## Company evidence capture
 
-`LIFECYCLE_COMPANY_CAPTURE_PROVIDER` defaults to `direct`, preserving the existing company-page fetch. To explicitly enable managed homepage capture, set it to exactly `firecrawl` and configure the server-only `FIRECRAWL_API_KEY`. Configuration is checked only when enrichment needs company evidence; it does not gate email delivery. Invalid configuration and provider failures use the existing enrichment retry handling, without a direct-fetch fallback.
+`LIFECYCLE_COMPANY_CAPTURE_PROVIDER` defaults to `direct`, preserving the existing company-page fetch. To use our self-hosted Firecrawl open-source browser scraper, set it to exactly `firecrawl`, configure `COMPANY_SCRAPER_URL` as its bare HTTPS origin, and supply the shared server-only `COMPANY_SCRAPER_SECRET`. These are our own service settings; no Firecrawl account or hosted API key is used. Explicit HTTP loopback IP origins are accepted for local container verification. Configuration is checked only when enrichment needs company evidence and does not gate email delivery. Failures use existing enrichment retry handling, without a direct-fetch fallback.
 
-Firecrawl capture makes one fresh homepage request with the basic proxy, a 10-second provider timeout, a 15-second total deadline, and a 2 MiB response limit. The existing HTML extractor produces the same bounded evidence schema. It accepts a changed final company hostname only when Firecrawl reports the requested source URL and a valid public HTTPS final URL. Local checks validate the input and final hostnames; Firecrawl owns remote DNS resolution and intermediate redirect safety. This is a provider trust boundary, not the direct fetcher's DNS-pinned transport.
+The client makes one homepage request with a 15-second total deadline and 2 MiB response limit. The scraper has a shorter 10-second work budget and one active capture; busy requests fail without queueing. The existing HTML extractor produces the same bounded evidence schema. The service returns the requested source and actual final browser URL; the client validates both and checks public input/final hostnames. The browser service owns remote navigation and subresource checks. This service boundary does not provide the direct fetcher's DNS-pinned transport guarantees, and capture is not proof of employment or company ownership. See [the scraper deployment](../../deployments/company-scraper/README.md) for its pinned source, patch, and verification commands.
 
-Capture logs contain provider, outcome, status, byte count, and reported credits where available (direct capture also identifies the fixed requested path). They exclude page text, company URLs, and credentials. Keep the default provider until an account key is configured and an authenticated capture is verified; public keyless experiments do not verify the production account integration.
+Client capture logs contain provider, outcome, status, and byte count where available (direct capture also identifies the fixed requested path). They exclude page text, company URLs, and credentials. Keep the default provider until the self-hosted service is deployed and authenticated capture is verified. Browser rendering does not include Firecrawl Cloud's advanced anti-bot engine.

apps/lifecycle/project.json

@@ -30,7 +30,8 @@
         "cwd": "apps/lifecycle",
         "commands": [
           "npx -y node@24 ../../node_modules/@dawn-ai/cli/dist/index.js check",
-          "npx -y node@24 ../../node_modules/typescript/bin/tsc --noEmit -p tsconfig.json"
+          "npx -y node@24 ../../node_modules/typescript/bin/tsc --noEmit -p tsconfig.json",
+          "npx -y node@24 --test ../../deployments/company-scraper/*.test.cjs"
         ],
         "parallel": false
       }

apps/lifecycle/src/enrichment/company-capture.spec.ts

@@ -57,10 +57,13 @@ describe('configured company capture', () => {
     const signal = new AbortController().signal;
     await createCompanyCapture({
       LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl',
-      FIRECRAWL_API_KEY: 'fixture-key',
+      COMPANY_SCRAPER_SECRET: 'fixture-key',
+      COMPANY_SCRAPER_URL: 'https://scraper.example.com',
     })('example.com', signal);
     expect(managed).toHaveBeenCalledWith('example.com', signal, {
-      apiKey: 'fixture-key',
+      secret: 'fixture-key',
+      serviceUrl: 'https://scraper.example.com',
+      allowLocalHttp: false,
       onDiagnostic: expect.any(Function),
     });
     expect(direct).not.toHaveBeenCalled();
@@ -82,7 +85,8 @@ describe('configured company capture', () => {
     async (key) => {
       const capture = createCompanyCapture({
         LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl',
-        FIRECRAWL_API_KEY: key,
+        COMPANY_SCRAPER_SECRET: key,
+        COMPANY_SCRAPER_URL: 'https://scraper.example.com',
       });
       await expect(
         capture('example.com', new AbortController().signal)
@@ -97,7 +101,8 @@ describe('configured company capture', () => {
     await expect(
       createCompanyCapture({
         LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl',
-        FIRECRAWL_API_KEY: 'fixture-key',
+        COMPANY_SCRAPER_SECRET: 'fixture-key',
+        COMPANY_SCRAPER_URL: 'https://scraper.example.com',
       })('example.com', new AbortController().signal)
     ).rejects.toThrow('firecrawl_provider_error');
     expect(managed).toHaveBeenCalledTimes(1);

apps/lifecycle/src/enrichment/company-capture.ts

@@ -23,14 +23,18 @@ export function createCompanyCapture(
           report({ provider: 'direct', ...diagnostic }),
       });
     } else if (provider === 'firecrawl') {
-      const apiKey = environment['FIRECRAWL_API_KEY']?.trim();
-      if (!apiKey) {
+      const secret = environment['COMPANY_SCRAPER_SECRET']?.trim();
+      if (!secret) {
         report({ provider: 'firecrawl', outcome: 'missing_key' });
         signal.throwIfAborted();
         throw new Error('company_capture_missing_key');
       }
       evidence = await fetchFirecrawlCompanyEvidence(domain, signal, {
-        apiKey,
+        secret,
+        serviceUrl: environment['COMPANY_SCRAPER_URL'] ?? '',
+        allowLocalHttp:
+          environment['NODE_ENV'] === 'development' ||
+          environment['NODE_ENV'] === 'test',
         onDiagnostic: report,
       });
     } else {

apps/lifecycle/src/enrichment/firecrawl.spec.ts

@@ -10,19 +10,21 @@ const html =
 const metadata = {
   sourceURL: 'https://example.com/',
   url: 'https://www.example.com/',
-  statusCode: 200,
+  pageStatusCode: 200,
 };
 const payload = (data = {}) => ({
-  success: true,
-  data: { html, metadata, ...data },
+  content: html,
+  ...metadata,
+  ...data,
 });
 function setup(body: unknown = payload()) {
   const fetch = vi
     .fn<typeof globalThis.fetch>()
     .mockResolvedValue(Response.json(body));
   const resolve = vi.fn().mockResolvedValue(['93.184.216.34']);
   const options: FirecrawlOptions = {
-    apiKey: 'test-key',
+    serviceUrl: 'https://scraper.example.com',
+    secret: 'test-key',
     fetch,
     resolve,
     now: () => new Date('2026-09-01T12:00:00Z'),
@@ -36,6 +38,37 @@ const run = (
 ) => fetchFirecrawlCompanyEvidence(domain, signal, options);
 afterEach(() => vi.useRealTimers());
 describe('Firecrawl homepage evidence', () => {
+  it.each([
+    'http://scraper.example.com',
+    'http://127.0.0.1:33003',
+    'https://user:password@scraper.example.com',
+    'https://scraper.example.com/path',
+    'https://scraper.example.com/?token=value',
+    'https://scraper.example.com/#hash',
+    'https://api.firecrawl.dev',
+    'https://scraper.example.com:444',
+  ])(
+    'rejects invalid service configuration %s before network',
+    async (serviceUrl) => {
+      const { options, fetch, resolve } = setup();
+      await expect(run({ ...options, serviceUrl })).rejects.toThrow(
+        'configuration'
+      );
+      expect(fetch).not.toHaveBeenCalled();
+      expect(resolve).not.toHaveBeenCalled();
+    }
+  );
+  it('permits an explicitly configured local loopback service', async () => {
+    const { options, fetch } = setup();
+    await expect(
+      run({
+        ...options,
+        serviceUrl: 'http://127.0.0.1:33003',
+        allowLocalHttp: true,
+      })
+    ).resolves.toHaveLength(1);
+    expect(fetch.mock.calls[0][0]).toBe('http://127.0.0.1:33003/scrape');
+  });
   it('uses exactly one fixed authenticated scrape and maps extracted HTML with actual URL provenance', async () => {
     const { fetch, options } = setup();
     const result = await run(options);
@@ -50,24 +83,19 @@ describe('Firecrawl homepage evidence', () => {
     ]);
     expect(fetch).toHaveBeenCalledTimes(1);
     const [url, init] = fetch.mock.calls[0];
-    expect(url).toBe('https://api.firecrawl.dev/v2/scrape');
+    expect(url).toBe('https://scraper.example.com/scrape');
     expect(init).toMatchObject({
       method: 'POST',
       redirect: 'error',
       headers: { authorization: 'Bearer test-key' },
     });
     expect(JSON.parse(String(init?.body))).toEqual({
       url: 'https://example.com/',
-      formats: ['html'],
-      onlyMainContent: true,
-      maxAge: 0,
-      timeout: 10000,
-      proxy: 'basic',
     });
   });
   it('requires the key before DNS or network', async () => {
     const { options, fetch, resolve } = setup();
-    await expect(run({ ...options, apiKey: ' ' })).rejects.toThrow(
+    await expect(run({ ...options, secret: ' ' })).rejects.toThrow(
       'configuration'
     );
     expect(fetch).not.toHaveBeenCalled();
@@ -91,7 +119,12 @@ describe('Firecrawl homepage evidence', () => {
   });
   it.each([
     { ...metadata, sourceURL: 'https://unrelated.com/' },
-    { sourceURL: metadata.sourceURL, ogUrl: metadata.url, statusCode: 200 },
+    {
+      sourceURL: metadata.sourceURL,
+      url: undefined,
+      ogUrl: metadata.url,
+      pageStatusCode: 200,
+    },
     { ...metadata, url: 'http://example.com/' },
     { ...metadata, url: 'https://127.0.0.1/' },
     { ...metadata, url: 'https://example.com/?token=secret' },
@@ -100,9 +133,7 @@ describe('Firecrawl homepage evidence', () => {
     { ...metadata, url: 'https://example.com:444/' },
     { ...metadata, url: `https://example.com/${'x'.repeat(500)}` },
   ])('rejects invalid provenance %#', async (invalid) => {
-    await expect(
-      run(setup(payload({ metadata: invalid })).options)
-    ).rejects.toThrow();
+    await expect(run(setup(payload(invalid)).options)).rejects.toThrow();
   });
   it('validates final hostname DNS', async () => {
     const { options, resolve } = setup();
@@ -112,9 +143,9 @@ describe('Firecrawl homepage evidence', () => {
     await expect(run(options)).rejects.toThrow('security_rejected');
   });
   it.each([
-    payload({ html: '' }),
-    payload({ metadata: { ...metadata, statusCode: 404 } }),
-    payload({ html: '<script>secret</script>' }),
+    payload({ content: '' }),
+    payload({ pageStatusCode: 404 }),
+    payload({ content: '<script>secret</script>' }),
   ])('returns no evidence for empty or missing pages %#', async (body) => {
     await expect(run(setup(body).options)).resolves.toEqual([]);
   });
@@ -138,7 +169,7 @@ describe('Firecrawl homepage evidence', () => {
     { success: true, data: null },
     {
       success: true,
-      data: { html, metadata: { ...metadata, statusCode: 503 } },
+      ...payload({ pageStatusCode: 503 }),
     },
   ])('rejects provider failures %#', async (body) => {
     await expect(run(setup(body).options)).rejects.toThrow();
@@ -281,7 +312,7 @@ describe('Firecrawl homepage evidence', () => {
       { length: 10 },
       (_, i) => `<p>${i}${'p'.repeat(300)}</p>`
     ).join('')}<style>private</style>`;
-    const result = await run(setup(payload({ html: body })).options);
+    const result = await run(setup(payload({ content: body })).options);
     expect(result[0].facts.length).toBeLessThanOrEqual(6);
     expect(result[0].snippets).toHaveLength(6);
     expect(
@@ -296,9 +327,7 @@ describe('Firecrawl homepage evidence', () => {
     await expect(run(options)).rejects.toThrow(/^invalid_response$/);
   });
   it('ignores observer errors and exposes only bounded diagnostic fields', async () => {
-    const { options } = setup(
-      payload({ metadata: { ...metadata, creditsUsed: 1, private: 'secret' } })
-    );
+    const { options } = setup(payload({ creditsUsed: 1, private: 'secret' }));
     const diagnostics: unknown[] = [];
     expect(
       await run({
@@ -316,7 +345,6 @@ describe('Firecrawl homepage evidence', () => {
         apiStatus: 200,
         pageStatus: 200,
         bytes: expect.any(Number),
-        credits: 1,
       },
     ]);
   });

apps/lifecycle/src/enrichment/firecrawl.ts

@@ -27,10 +27,11 @@ export interface FirecrawlDiagnostic {
   apiStatus?: number;
   pageStatus?: number;
   bytes?: number;
-  credits?: number;
 }
 export interface FirecrawlOptions {
-  apiKey: string;
+  serviceUrl: string;
+  secret: string;
+  allowLocalHttp?: boolean;
   fetch?: typeof fetch;
   resolve?: CompanyFetchDependencies['resolve'];
   now?: () => Date;
@@ -43,6 +44,33 @@ class FirecrawlError extends Error {
   }
 }
 
+function serviceEndpoint(value: string, allowLocalHttp = false): string {
+  try {
+    const url = new URL(value);
+    const loopback =
+      allowLocalHttp &&
+      url.protocol === 'http:' &&
+      (url.hostname === '127.0.0.1' || url.hostname === '[::1]');
+    if (
+      value !== value.trim() ||
+      url.username ||
+      url.password ||
+      url.pathname !== '/' ||
+      url.search ||
+      url.hash ||
+      value.includes('?') ||
+      value.includes('#') ||
+      url.hostname === 'api.firecrawl.dev' ||
+      (!loopback && (url.protocol !== 'https:' || url.port))
+    ) {
+      throw new Error();
+    }
+    return new URL('/scrape', url).toString();
+  } catch {
+    throw new FirecrawlError('configuration');
+  }
+}
+
 // Race every asynchronous stage, including injected transports and stalled bodies.
 function abortable<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
   signal.throwIfAborted();
@@ -155,34 +183,31 @@ export async function fetchFirecrawlCompanyEvidence(
   };
   try {
     combined.throwIfAborted();
-    if (!options.apiKey?.trim()) throw new FirecrawlError('configuration');
+    if (!options.secret || /\s/u.test(options.secret))
+      throw new FirecrawlError('configuration');
+    const endpoint = serviceEndpoint(
+      options.serviceUrl,
+      options.allowLocalHttp
+    );
     const hostname = await abortable(
       validatePublicCompanyHostname(domain, combined, options.resolve),
       combined
     );
     combined.throwIfAborted();
     const requestedUrl = `https://${hostname}/`;
-    const pending = (options.fetch ?? fetch)(
-      'https://api.firecrawl.dev/v2/scrape',
-      {
-        method: 'POST',
-        redirect: 'error',
-        signal: combined,
-        headers: {
-          authorization: `Bearer ${options.apiKey}`,
-          'content-type': 'application/json',
-          accept: 'application/json',
-        },
-        body: JSON.stringify({
-          url: requestedUrl,
-          formats: ['html'],
-          onlyMainContent: true,
-          maxAge: 0,
-          timeout: 10000,
-          proxy: 'basic',
-        }),
-      }
-    );
+    const pending = (options.fetch ?? fetch)(endpoint, {
+      method: 'POST',
+      redirect: 'error',
+      signal: combined,
+      headers: {
+        authorization: `Bearer ${options.secret}`,
+        'content-type': 'application/json',
+        accept: 'application/json',
+      },
+      body: JSON.stringify({
+        url: requestedUrl,
+      }),
+    });
     void pending.then(
       (response) => {
         if (combined.aborted) dispose(response.body);
@@ -200,19 +225,16 @@ export async function fetchFirecrawlCompanyEvidence(
     }
     const result = record(await boundedJson(response, combined, diagnostic));
     combined.throwIfAborted();
-    if (result.success !== true) throw new FirecrawlError('invalid_response');
-    const data = record(result.data);
-    const metadata = record(data.metadata);
-    const source = safeUrl(metadata.sourceURL);
-    const final = safeUrl(metadata.url);
+    const source = safeUrl(result.sourceURL);
+    const final = safeUrl(result.url);
     if (source.toString() !== requestedUrl)
       throw new FirecrawlError('invalid_provenance');
     await abortable(
       validatePublicCompanyHostname(final.hostname, combined, options.resolve),
       combined
     );
     combined.throwIfAborted();
-    const status = metadata.statusCode;
+    const status = result.pageStatusCode;
     if (
       typeof status !== 'number' ||
       !Number.isInteger(status) ||
@@ -221,12 +243,6 @@ export async function fetchFirecrawlCompanyEvidence(
     )
       throw new FirecrawlError('invalid_response');
     diagnostic.pageStatus = status;
-    if (
-      typeof metadata.creditsUsed === 'number' &&
-      Number.isSafeInteger(metadata.creditsUsed) &&
-      metadata.creditsUsed >= 0
-    )
-      diagnostic.credits = metadata.creditsUsed;
     if (status !== 404 && (status < 200 || status >= 300))
       throw new FirecrawlError('page_http_error');
     if (status === 404) {
@@ -235,9 +251,9 @@ export async function fetchFirecrawlCompanyEvidence(
       combined.throwIfAborted();
       return [];
     }
-    if (typeof data.html !== 'string')
+    if (typeof result.content !== 'string')
       throw new FirecrawlError('invalid_response');
-    const body = Buffer.from(data.html, 'utf8');
+    const body = Buffer.from(result.content, 'utf8');
     const extracted = extractEvidence(body);
     if (!extracted.facts.length && !extracted.snippets.length) {
       diagnostic.outcome = 'no_evidence';