diff --git a/apps/lifecycle/README.md b/apps/lifecycle/README.md index 242a374ec..b3d924da6 100644 --- a/apps/lifecycle/README.md +++ b/apps/lifecycle/README.md @@ -29,8 +29,8 @@ Use [DOGFOOD.md](./DOGFOOD.md) for the provider-free setup, probe, and exact cle ## Company evidence capture -`LIFECYCLE_COMPANY_CAPTURE_PROVIDER` defaults to `direct`, preserving the existing company-page fetch. To explicitly enable managed homepage capture, set it to exactly `firecrawl` and configure the server-only `FIRECRAWL_API_KEY`. Configuration is checked only when enrichment needs company evidence; it does not gate email delivery. Invalid configuration and provider failures use the existing enrichment retry handling, without a direct-fetch fallback. +`LIFECYCLE_COMPANY_CAPTURE_PROVIDER` defaults to `direct`, preserving the existing company-page fetch. To use our self-hosted Firecrawl open-source browser scraper, set it to exactly `firecrawl`, configure `COMPANY_SCRAPER_URL` as its bare HTTPS origin, and supply the shared server-only `COMPANY_SCRAPER_SECRET`. These are our own service settings; no Firecrawl account or hosted API key is used. Explicit HTTP loopback IP origins are accepted for local container verification. Configuration is checked only when enrichment needs company evidence and does not gate email delivery. Failures use existing enrichment retry handling, without a direct-fetch fallback. -Firecrawl capture makes one fresh homepage request with the basic proxy, a 10-second provider timeout, a 15-second total deadline, and a 2 MiB response limit. The existing HTML extractor produces the same bounded evidence schema. It accepts a changed final company hostname only when Firecrawl reports the requested source URL and a valid public HTTPS final URL. Local checks validate the input and final hostnames; Firecrawl owns remote DNS resolution and intermediate redirect safety. This is a provider trust boundary, not the direct fetcher's DNS-pinned transport. +The client makes one homepage request with a 15-second total deadline and 2 MiB response limit. The scraper has a shorter 10-second work budget and one active capture; busy requests fail without queueing. The existing HTML extractor produces the same bounded evidence schema. The service returns the requested source and actual final browser URL; the client validates both and checks public input/final hostnames. The browser service owns remote navigation and subresource checks. This service boundary does not provide the direct fetcher's DNS-pinned transport guarantees, and capture is not proof of employment or company ownership. See [the scraper deployment](../../deployments/company-scraper/README.md) for its pinned source, patch, and verification commands. -Capture logs contain provider, outcome, status, byte count, and reported credits where available (direct capture also identifies the fixed requested path). They exclude page text, company URLs, and credentials. Keep the default provider until an account key is configured and an authenticated capture is verified; public keyless experiments do not verify the production account integration. +Client capture logs contain provider, outcome, status, and byte count where available (direct capture also identifies the fixed requested path). They exclude page text, company URLs, and credentials. Keep the default provider until the self-hosted service is deployed and authenticated capture is verified. Browser rendering does not include Firecrawl Cloud's advanced anti-bot engine. diff --git a/apps/lifecycle/project.json b/apps/lifecycle/project.json index 6515341bd..19f63c68b 100644 --- a/apps/lifecycle/project.json +++ b/apps/lifecycle/project.json @@ -30,7 +30,8 @@ "cwd": "apps/lifecycle", "commands": [ "npx -y node@24 ../../node_modules/@dawn-ai/cli/dist/index.js check", - "npx -y node@24 ../../node_modules/typescript/bin/tsc --noEmit -p tsconfig.json" + "npx -y node@24 ../../node_modules/typescript/bin/tsc --noEmit -p tsconfig.json", + "npx -y node@24 --test ../../deployments/company-scraper/*.test.cjs" ], "parallel": false } diff --git a/apps/lifecycle/src/enrichment/company-capture.spec.ts b/apps/lifecycle/src/enrichment/company-capture.spec.ts index f77c707d3..bdf503747 100644 --- a/apps/lifecycle/src/enrichment/company-capture.spec.ts +++ b/apps/lifecycle/src/enrichment/company-capture.spec.ts @@ -57,10 +57,13 @@ describe('configured company capture', () => { const signal = new AbortController().signal; await createCompanyCapture({ LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl', - FIRECRAWL_API_KEY: 'fixture-key', + COMPANY_SCRAPER_SECRET: 'fixture-key', + COMPANY_SCRAPER_URL: 'https://scraper.example.com', })('example.com', signal); expect(managed).toHaveBeenCalledWith('example.com', signal, { - apiKey: 'fixture-key', + secret: 'fixture-key', + serviceUrl: 'https://scraper.example.com', + allowLocalHttp: false, onDiagnostic: expect.any(Function), }); expect(direct).not.toHaveBeenCalled(); @@ -82,7 +85,8 @@ describe('configured company capture', () => { async (key) => { const capture = createCompanyCapture({ LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl', - FIRECRAWL_API_KEY: key, + COMPANY_SCRAPER_SECRET: key, + COMPANY_SCRAPER_URL: 'https://scraper.example.com', }); await expect( capture('example.com', new AbortController().signal) @@ -97,7 +101,8 @@ describe('configured company capture', () => { await expect( createCompanyCapture({ LIFECYCLE_COMPANY_CAPTURE_PROVIDER: 'firecrawl', - FIRECRAWL_API_KEY: 'fixture-key', + COMPANY_SCRAPER_SECRET: 'fixture-key', + COMPANY_SCRAPER_URL: 'https://scraper.example.com', })('example.com', new AbortController().signal) ).rejects.toThrow('firecrawl_provider_error'); expect(managed).toHaveBeenCalledTimes(1); diff --git a/apps/lifecycle/src/enrichment/company-capture.ts b/apps/lifecycle/src/enrichment/company-capture.ts index d35495f4b..d656fec48 100644 --- a/apps/lifecycle/src/enrichment/company-capture.ts +++ b/apps/lifecycle/src/enrichment/company-capture.ts @@ -23,14 +23,18 @@ export function createCompanyCapture( report({ provider: 'direct', ...diagnostic }), }); } else if (provider === 'firecrawl') { - const apiKey = environment['FIRECRAWL_API_KEY']?.trim(); - if (!apiKey) { + const secret = environment['COMPANY_SCRAPER_SECRET']?.trim(); + if (!secret) { report({ provider: 'firecrawl', outcome: 'missing_key' }); signal.throwIfAborted(); throw new Error('company_capture_missing_key'); } evidence = await fetchFirecrawlCompanyEvidence(domain, signal, { - apiKey, + secret, + serviceUrl: environment['COMPANY_SCRAPER_URL'] ?? '', + allowLocalHttp: + environment['NODE_ENV'] === 'development' || + environment['NODE_ENV'] === 'test', onDiagnostic: report, }); } else { diff --git a/apps/lifecycle/src/enrichment/firecrawl.spec.ts b/apps/lifecycle/src/enrichment/firecrawl.spec.ts index 70995a554..814a31cef 100644 --- a/apps/lifecycle/src/enrichment/firecrawl.spec.ts +++ b/apps/lifecycle/src/enrichment/firecrawl.spec.ts @@ -10,11 +10,12 @@ const html = const metadata = { sourceURL: 'https://example.com/', url: 'https://www.example.com/', - statusCode: 200, + pageStatusCode: 200, }; const payload = (data = {}) => ({ - success: true, - data: { html, metadata, ...data }, + content: html, + ...metadata, + ...data, }); function setup(body: unknown = payload()) { const fetch = vi @@ -22,7 +23,8 @@ function setup(body: unknown = payload()) { .mockResolvedValue(Response.json(body)); const resolve = vi.fn().mockResolvedValue(['93.184.216.34']); const options: FirecrawlOptions = { - apiKey: 'test-key', + serviceUrl: 'https://scraper.example.com', + secret: 'test-key', fetch, resolve, now: () => new Date('2026-09-01T12:00:00Z'), @@ -36,6 +38,37 @@ const run = ( ) => fetchFirecrawlCompanyEvidence(domain, signal, options); afterEach(() => vi.useRealTimers()); describe('Firecrawl homepage evidence', () => { + it.each([ + 'http://scraper.example.com', + 'http://127.0.0.1:33003', + 'https://user:password@scraper.example.com', + 'https://scraper.example.com/path', + 'https://scraper.example.com/?token=value', + 'https://scraper.example.com/#hash', + 'https://api.firecrawl.dev', + 'https://scraper.example.com:444', + ])( + 'rejects invalid service configuration %s before network', + async (serviceUrl) => { + const { options, fetch, resolve } = setup(); + await expect(run({ ...options, serviceUrl })).rejects.toThrow( + 'configuration' + ); + expect(fetch).not.toHaveBeenCalled(); + expect(resolve).not.toHaveBeenCalled(); + } + ); + it('permits an explicitly configured local loopback service', async () => { + const { options, fetch } = setup(); + await expect( + run({ + ...options, + serviceUrl: 'http://127.0.0.1:33003', + allowLocalHttp: true, + }) + ).resolves.toHaveLength(1); + expect(fetch.mock.calls[0][0]).toBe('http://127.0.0.1:33003/scrape'); + }); it('uses exactly one fixed authenticated scrape and maps extracted HTML with actual URL provenance', async () => { const { fetch, options } = setup(); const result = await run(options); @@ -50,7 +83,7 @@ describe('Firecrawl homepage evidence', () => { ]); expect(fetch).toHaveBeenCalledTimes(1); const [url, init] = fetch.mock.calls[0]; - expect(url).toBe('https://api.firecrawl.dev/v2/scrape'); + expect(url).toBe('https://scraper.example.com/scrape'); expect(init).toMatchObject({ method: 'POST', redirect: 'error', @@ -58,16 +91,11 @@ describe('Firecrawl homepage evidence', () => { }); expect(JSON.parse(String(init?.body))).toEqual({ url: 'https://example.com/', - formats: ['html'], - onlyMainContent: true, - maxAge: 0, - timeout: 10000, - proxy: 'basic', }); }); it('requires the key before DNS or network', async () => { const { options, fetch, resolve } = setup(); - await expect(run({ ...options, apiKey: ' ' })).rejects.toThrow( + await expect(run({ ...options, secret: ' ' })).rejects.toThrow( 'configuration' ); expect(fetch).not.toHaveBeenCalled(); @@ -91,7 +119,12 @@ describe('Firecrawl homepage evidence', () => { }); it.each([ { ...metadata, sourceURL: 'https://unrelated.com/' }, - { sourceURL: metadata.sourceURL, ogUrl: metadata.url, statusCode: 200 }, + { + sourceURL: metadata.sourceURL, + url: undefined, + ogUrl: metadata.url, + pageStatusCode: 200, + }, { ...metadata, url: 'http://example.com/' }, { ...metadata, url: 'https://127.0.0.1/' }, { ...metadata, url: 'https://example.com/?token=secret' }, @@ -100,9 +133,7 @@ describe('Firecrawl homepage evidence', () => { { ...metadata, url: 'https://example.com:444/' }, { ...metadata, url: `https://example.com/${'x'.repeat(500)}` }, ])('rejects invalid provenance %#', async (invalid) => { - await expect( - run(setup(payload({ metadata: invalid })).options) - ).rejects.toThrow(); + await expect(run(setup(payload(invalid)).options)).rejects.toThrow(); }); it('validates final hostname DNS', async () => { const { options, resolve } = setup(); @@ -112,9 +143,9 @@ describe('Firecrawl homepage evidence', () => { await expect(run(options)).rejects.toThrow('security_rejected'); }); it.each([ - payload({ html: '' }), - payload({ metadata: { ...metadata, statusCode: 404 } }), - payload({ html: '' }), + payload({ content: '' }), + payload({ pageStatusCode: 404 }), + payload({ content: '' }), ])('returns no evidence for empty or missing pages %#', async (body) => { await expect(run(setup(body).options)).resolves.toEqual([]); }); @@ -138,7 +169,7 @@ describe('Firecrawl homepage evidence', () => { { success: true, data: null }, { success: true, - data: { html, metadata: { ...metadata, statusCode: 503 } }, + ...payload({ pageStatusCode: 503 }), }, ])('rejects provider failures %#', async (body) => { await expect(run(setup(body).options)).rejects.toThrow(); @@ -281,7 +312,7 @@ describe('Firecrawl homepage evidence', () => { { length: 10 }, (_, i) => `

${i}${'p'.repeat(300)}

` ).join('')}`; - const result = await run(setup(payload({ html: body })).options); + const result = await run(setup(payload({ content: body })).options); expect(result[0].facts.length).toBeLessThanOrEqual(6); expect(result[0].snippets).toHaveLength(6); expect( @@ -296,9 +327,7 @@ describe('Firecrawl homepage evidence', () => { await expect(run(options)).rejects.toThrow(/^invalid_response$/); }); it('ignores observer errors and exposes only bounded diagnostic fields', async () => { - const { options } = setup( - payload({ metadata: { ...metadata, creditsUsed: 1, private: 'secret' } }) - ); + const { options } = setup(payload({ creditsUsed: 1, private: 'secret' })); const diagnostics: unknown[] = []; expect( await run({ @@ -316,7 +345,6 @@ describe('Firecrawl homepage evidence', () => { apiStatus: 200, pageStatus: 200, bytes: expect.any(Number), - credits: 1, }, ]); }); diff --git a/apps/lifecycle/src/enrichment/firecrawl.ts b/apps/lifecycle/src/enrichment/firecrawl.ts index 3117202b5..d93183f45 100644 --- a/apps/lifecycle/src/enrichment/firecrawl.ts +++ b/apps/lifecycle/src/enrichment/firecrawl.ts @@ -27,10 +27,11 @@ export interface FirecrawlDiagnostic { apiStatus?: number; pageStatus?: number; bytes?: number; - credits?: number; } export interface FirecrawlOptions { - apiKey: string; + serviceUrl: string; + secret: string; + allowLocalHttp?: boolean; fetch?: typeof fetch; resolve?: CompanyFetchDependencies['resolve']; now?: () => Date; @@ -43,6 +44,33 @@ class FirecrawlError extends Error { } } +function serviceEndpoint(value: string, allowLocalHttp = false): string { + try { + const url = new URL(value); + const loopback = + allowLocalHttp && + url.protocol === 'http:' && + (url.hostname === '127.0.0.1' || url.hostname === '[::1]'); + if ( + value !== value.trim() || + url.username || + url.password || + url.pathname !== '/' || + url.search || + url.hash || + value.includes('?') || + value.includes('#') || + url.hostname === 'api.firecrawl.dev' || + (!loopback && (url.protocol !== 'https:' || url.port)) + ) { + throw new Error(); + } + return new URL('/scrape', url).toString(); + } catch { + throw new FirecrawlError('configuration'); + } +} + // Race every asynchronous stage, including injected transports and stalled bodies. function abortable(promise: Promise, signal: AbortSignal): Promise { signal.throwIfAborted(); @@ -155,34 +183,31 @@ export async function fetchFirecrawlCompanyEvidence( }; try { combined.throwIfAborted(); - if (!options.apiKey?.trim()) throw new FirecrawlError('configuration'); + if (!options.secret || /\s/u.test(options.secret)) + throw new FirecrawlError('configuration'); + const endpoint = serviceEndpoint( + options.serviceUrl, + options.allowLocalHttp + ); const hostname = await abortable( validatePublicCompanyHostname(domain, combined, options.resolve), combined ); combined.throwIfAborted(); const requestedUrl = `https://${hostname}/`; - const pending = (options.fetch ?? fetch)( - 'https://api.firecrawl.dev/v2/scrape', - { - method: 'POST', - redirect: 'error', - signal: combined, - headers: { - authorization: `Bearer ${options.apiKey}`, - 'content-type': 'application/json', - accept: 'application/json', - }, - body: JSON.stringify({ - url: requestedUrl, - formats: ['html'], - onlyMainContent: true, - maxAge: 0, - timeout: 10000, - proxy: 'basic', - }), - } - ); + const pending = (options.fetch ?? fetch)(endpoint, { + method: 'POST', + redirect: 'error', + signal: combined, + headers: { + authorization: `Bearer ${options.secret}`, + 'content-type': 'application/json', + accept: 'application/json', + }, + body: JSON.stringify({ + url: requestedUrl, + }), + }); void pending.then( (response) => { if (combined.aborted) dispose(response.body); @@ -200,11 +225,8 @@ export async function fetchFirecrawlCompanyEvidence( } const result = record(await boundedJson(response, combined, diagnostic)); combined.throwIfAborted(); - if (result.success !== true) throw new FirecrawlError('invalid_response'); - const data = record(result.data); - const metadata = record(data.metadata); - const source = safeUrl(metadata.sourceURL); - const final = safeUrl(metadata.url); + const source = safeUrl(result.sourceURL); + const final = safeUrl(result.url); if (source.toString() !== requestedUrl) throw new FirecrawlError('invalid_provenance'); await abortable( @@ -212,7 +234,7 @@ export async function fetchFirecrawlCompanyEvidence( combined ); combined.throwIfAborted(); - const status = metadata.statusCode; + const status = result.pageStatusCode; if ( typeof status !== 'number' || !Number.isInteger(status) || @@ -221,12 +243,6 @@ export async function fetchFirecrawlCompanyEvidence( ) throw new FirecrawlError('invalid_response'); diagnostic.pageStatus = status; - if ( - typeof metadata.creditsUsed === 'number' && - Number.isSafeInteger(metadata.creditsUsed) && - metadata.creditsUsed >= 0 - ) - diagnostic.credits = metadata.creditsUsed; if (status !== 404 && (status < 200 || status >= 300)) throw new FirecrawlError('page_http_error'); if (status === 404) { @@ -235,9 +251,9 @@ export async function fetchFirecrawlCompanyEvidence( combined.throwIfAborted(); return []; } - if (typeof data.html !== 'string') + if (typeof result.content !== 'string') throw new FirecrawlError('invalid_response'); - const body = Buffer.from(data.html, 'utf8'); + const body = Buffer.from(result.content, 'utf8'); const extracted = extractEvidence(body); if (!extracted.facts.length && !extracted.snippets.length) { diagnostic.outcome = 'no_evidence'; diff --git a/deployments/company-scraper/.dockerignore b/deployments/company-scraper/.dockerignore new file mode 100644 index 000000000..40809c75d --- /dev/null +++ b/deployments/company-scraper/.dockerignore @@ -0,0 +1,4 @@ +* +!Dockerfile +!upstream.patch +!company-handler.cjs diff --git a/deployments/company-scraper/.gitattributes b/deployments/company-scraper/.gitattributes new file mode 100644 index 000000000..db2ac6b89 --- /dev/null +++ b/deployments/company-scraper/.gitattributes @@ -0,0 +1,2 @@ +# Unified diff context uses a single space for blank source lines. +upstream.patch -whitespace diff --git a/deployments/company-scraper/Dockerfile b/deployments/company-scraper/Dockerfile new file mode 100644 index 000000000..1b3e028bc --- /dev/null +++ b/deployments/company-scraper/Dockerfile @@ -0,0 +1,26 @@ +FROM node:22-slim AS source +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /upstream +RUN git init && git remote add origin https://github.com/firecrawl/firecrawl.git \ + && git fetch --depth 1 origin 4cee46d827e1353e66228aa7f6c53734581b8f7b \ + && git checkout --detach FETCH_HEAD +COPY upstream.patch /tmp/upstream.patch +RUN git apply --check /tmp/upstream.patch && git apply /tmp/upstream.patch + +FROM node:22-slim +ENV PNPM_HOME=/pnpm PATH=/pnpm:$PATH CI=true \ + PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/playwright PORT=3003 +RUN corepack enable && corepack prepare pnpm@11.4.0 --activate +WORKDIR /usr/src/app +COPY --from=source /upstream/apps/playwright-service-ts/ ./ +COPY --from=source /upstream/LICENSE /usr/src/app/UPSTREAM-LICENSE +RUN pnpm install --frozen-lockfile \ + && pnpm exec playwright install chromium --with-deps \ + && chmod -R a+rX /usr/local/share/playwright +COPY company-handler.cjs ./ +RUN pnpm run build && cp company-handler.cjs dist/company-handler.cjs +ENV BLOCK_MEDIA=true +USER node +EXPOSE 3003 +CMD ["node", "dist/api.js"] diff --git a/deployments/company-scraper/README.md b/deployments/company-scraper/README.md new file mode 100644 index 000000000..663869bc5 --- /dev/null +++ b/deployments/company-scraper/README.md @@ -0,0 +1,17 @@ +# Company scraper + +This packages Firecrawl's actual standalone `apps/playwright-service-ts` component at commit `4cee46d827e1353e66228aa7f6c53734581b8f7b` from https://github.com/firecrawl/firecrawl. The build retains upstream source and root license (`UPSTREAM-LICENSE`); the upstream component package lists Jeff Pereira and ISC. Review upstream licensing for your distribution. `upstream.patch` narrows the HTTP interface while retaining upstream browser setup, navigation, extraction, request interception and SSRF proxy checks. `company-handler.cjs` owns admission, authentication and cancellation policy. + +Build from this directory: `docker build -t threadplane-company-scraper .` + +Run with a securely supplied `COMPANY_SCRAPER_SECRET` environment variable, `--cpus=2 --memory=2g --cap-drop=ALL --security-opt=no-new-privileges --init -p 127.0.0.1:3003:3003`. The image runs as the existing nonroot `node` user. Publish behind HTTPS; never expose the local port publicly. The upstream Chromium launch uses `--no-sandbox`, so keep the container isolated from internal services and apply platform egress restrictions where supported. + +`GET /health` is unauthenticated and reports browser connection readiness. `POST /scrape` requires `Authorization: Bearer ` and exactly `{ "url": "https://example.com/" }`. Inputs must be HTTPS homepages without credentials, nonstandard ports, paths, queries or fragments. Public DNS checks apply at input, navigation/subresource interception, proxy connection and final URL. Browser DNS is not pinned to the initial lookup; this is an authenticated service trust boundary, not a claim of DNS rebinding protection equivalent to a pinned direct fetch. + +Success returns `{ content, pageStatusCode, sourceURL, url }`, with the requested source and actual final browser URL. One active request is accepted; additional requests fail immediately with 503. Work has a ten-second deadline covering DNS, context creation/setup, navigation and extraction. Deadline or client disconnect initiates context closure, including contexts allocated late. The serialized response cap is 2 MiB. The user agent is fixed to `ThreadplaneCompanyResearch/1.0`; headers, cookies, TLS overrides and other caller options are rejected. Error responses and service logs omit raw URLs, headers and exception values. Startup requires the secret. + +Context cleanup retains the admission slot until closure completes. If browser allocation or closure never settles, subsequent requests fail closed with 503; restart the container. Health reports browser connection only, not capacity or a probe capture. + +Capture reads the document after `DOMContentLoaded`, without waiting for every page asset to finish loading. Content added asynchronously after that point may be absent. + +Run deterministic policy tests from the repository root: `node --test deployments/company-scraper/*.test.cjs`. Run `check-upstream-security.cjs` inside the built image (command in the file) to execute the patched upstream navigation, subresource and proxy checks with controlled public/private DNS answers, including an attempted environment bypass. No hosted Firecrawl account, model keys, queue, database or worker service is needed. diff --git a/deployments/company-scraper/check-upstream-security.cjs b/deployments/company-scraper/check-upstream-security.cjs new file mode 100644 index 000000000..ed02c0101 --- /dev/null +++ b/deployments/company-scraper/check-upstream-security.cjs @@ -0,0 +1,154 @@ +// Execute against the actual patched upstream source inside the built image: +// docker run --rm --entrypoint node -v "$PWD/check-upstream-security.cjs:/tmp/check.cjs:ro" IMAGE /tmp/check.cjs +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const { createRequire } = require('node:module'); +const upstreamRequire = createRequire('/usr/src/app/api.ts'); +const ts = upstreamRequire('typescript'); +let routeHandler, proxyCheck; +const fakeContext = { + route: async (pattern, handler) => { + if (pattern === '**/*') routeHandler = handler; + }, +}; +const fakeBrowser = { newContext: async () => fakeContext }; +const source = + fs + .readFileSync('/usr/src/app/api.ts', 'utf8') + .replace(/start\(\)\.catch\(\(error\) => \{[\s\S]*?\n\}\);/, '') + + '\nmodule.exports = {initializeBrowser,createContext,startSSRFProxy,assertSafeTargetUrl,scrapePage};'; +const compiled = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2016, + esModuleInterop: true, + }, +}).outputText; +const exportsObject = {}; +const context = { + exports: exportsObject, + module: { exports: exportsObject }, + console, + URL, + process: { + env: { COMPANY_SCRAPER_SECRET: 'test-only', ALLOW_LOCAL_WEBHOOKS: 'true' }, + }, + require: (name) => { + if (name === 'playwright') + return { chromium: { launch: async () => fakeBrowser } }; + if (name === 'dotenv') return { config: () => {} }; + if (name === 'dns/promises') + return { + lookup: async (host) => [ + { + address: host === 'public.example' ? '93.184.216.34' : '127.0.0.1', + }, + ], + }; + if (name === 'proxy-chain') + return { + Server: class { + constructor(options) { + proxyCheck = options.prepareRequestFunction; + this.port = 1234; + } + async listen() {} + }, + RequestError: class extends Error {}, + }; + if (name === './company-handler.cjs') + return upstreamRequire('/usr/src/app/company-handler.cjs'); + return upstreamRequire(name); + }, +}; +vm.runInNewContext(compiled, context); +(async () => { + const api = context.module.exports; + await api.initializeBrowser(); + const bundle = await api.createContext( + false, + 'ThreadplaneCompanyResearch/1.0' + ); + for (const navigation of [false, true]) { + let aborted = false, + continued = false; + await routeHandler( + { + abort: async () => { + aborted = true; + }, + continue: async () => { + continued = true; + }, + }, + { + url: () => 'http://private.example/resource', + isNavigationRequest: () => navigation, + } + ); + assert.equal(aborted, true); + assert.equal(continued, false); + } + assert.equal( + bundle.securityState.blockedNavigationRequestUrl, + 'http://private.example/resource' + ); + let continued = false; + await routeHandler( + { + abort: async () => assert.fail('public blocked'), + continue: async () => { + continued = true; + }, + }, + { url: () => 'https://public.example/', isNavigationRequest: () => true } + ); + assert.equal(continued, true); + await api.startSSRFProxy(); + await assert.rejects(() => proxyCheck({ hostname: 'private.example' })); + await proxyCheck({ hostname: 'public.example' }); + await assert.rejects(() => api.assertSafeTargetUrl('http://127.0.0.1/')); + const document = await api.scrapePage( + { + goto: async (_url, options) => { + assert.equal(options.waitUntil, 'domcontentloaded'); + return null; + }, + content: async () => 'document ready before full asset load', + }, + 'https://public.example/', + 'domcontentloaded', + 0, + 100, + undefined, + { blockedNavigationRequestUrl: null } + ); + assert.equal( + document.content, + 'document ready before full asset load' + ); + await assert.rejects( + () => + api.scrapePage( + { + goto: async () => { + throw Error('navigation failed'); + }, + }, + 'https://public.example/', + 'load', + 0, + 100, + undefined, + bundle.securityState + ), + /navigation to private/ + ); + console.log( + 'Patched upstream navigation, subresource and proxy private-address guards passed' + ); +})().catch((error) => { + console.error('Upstream security check failed', error); + process.exitCode = 1; +}); diff --git a/deployments/company-scraper/company-handler.cjs b/deployments/company-scraper/company-handler.cjs new file mode 100644 index 000000000..e09d240ed --- /dev/null +++ b/deployments/company-scraper/company-handler.cjs @@ -0,0 +1,175 @@ +// HTTP policy for the pinned Firecrawl standalone Playwright component. +// Browser creation, navigation, content extraction and SSRF checks remain upstream. +const { timingSafeEqual } = require('node:crypto'); +const MAX_RESPONSE_BYTES = 2 * 1024 * 1024; + +function createHandler({ + secret, + ready, + assertSafeTargetUrl, + createContext, + scrapePage, + budgetMs = 10000, +}) { + if (!secret || !secret.trim()) + throw new Error('COMPANY_SCRAPER_SECRET is required'); + const expected = Buffer.from(`Bearer ${secret}`); + let busy = false; + return async (req, res) => { + const supplied = Buffer.from(req.get('authorization') || ''); + if ( + supplied.length !== expected.length || + !timingSafeEqual(supplied, expected) + ) + return res.status(401).json({ error: 'unauthorized' }); + if (!ready() || busy) return res.status(503).json({ error: 'unavailable' }); + busy = true; + const deadline = performance.now() + budgetMs; + let stopped = false, + disconnected = false, + context, + closing, + allocationPending = false; + const close = () => { + if (context && !closing) + closing = Promise.resolve() + .then(() => context.close()) + .catch(() => {}); + return closing; + }; + let rejectStop; + const stopPromise = new Promise((_, reject) => { + rejectStop = reject; + }); + // A client can already be disconnected before the first asynchronous stage. + stopPromise.catch(() => {}); + const stop = (code) => { + if (stopped) return; + stopped = true; + close(); + rejectStop(Object.assign(new Error(code), { code })); + }; + const timer = setTimeout(() => stop('deadline'), budgetMs); + const disconnect = () => { + if (!res.writableEnded) { + disconnected = true; + stop('disconnect'); + } + }; + req.once('aborted', disconnect); + res.once('close', disconnect); + if (req.aborted || res.destroyed) disconnect(); + const stage = async (operation) => { + if (stopped) + throw Object.assign(new Error('deadline'), { code: 'deadline' }); + return Promise.race([ + Promise.resolve().then(() => { + if (stopped) + throw Object.assign(new Error('deadline'), { code: 'deadline' }); + return operation(); + }), + stopPromise, + ]); + }; + try { + const body = req.body; + let input; + try { + if ( + !body || + Array.isArray(body) || + Object.keys(body).length !== 1 || + typeof body.url !== 'string' || + body.url.length > 500 || + body.url.trim() !== body.url || + /[?#]/.test(body.url) + ) + throw Error(); + input = new URL(body.url); + if ( + input.protocol !== 'https:' || + input.username || + input.password || + input.port || + input.pathname !== '/' || + input.search || + input.hash + ) + throw Error(); + } catch { + return res.status(400).json({ error: 'invalid_request' }); + } + await stage(() => assertSafeTargetUrl(body.url)); + const bundle = await stage(async () => { + allocationPending = true; + try { + return await createContext( + false, + 'ThreadplaneCompanyResearch/1.0', + (allocated) => { + context = allocated; + allocationPending = false; + if (stopped) + close().then(() => { + busy = false; + }); + } + ); + } finally { + allocationPending = false; + if (stopped && !context) busy = false; + } + }); + const page = await stage(() => bundle.context.newPage()); + const result = await stage(() => + scrapePage( + page, + body.url, + 'domcontentloaded', + 0, + budgetMs, + undefined, + bundle.securityState + ) + ); + const finalUrl = page.url(); + const parsed = new URL(finalUrl); + if ( + finalUrl.length > 500 || + /[?#]/.test(finalUrl) || + parsed.protocol !== 'https:' || + parsed.username || + parsed.password || + parsed.port + ) + throw Error(); + await stage(() => assertSafeTargetUrl(finalUrl)); + const serialized = JSON.stringify({ + content: result.content, + pageStatusCode: result.status, + sourceURL: body.url, + url: finalUrl, + }); + if (Buffer.byteLength(serialized) > MAX_RESPONSE_BYTES) throw Error(); + if (performance.now() >= deadline) + throw Object.assign(new Error('deadline'), { code: 'deadline' }); + if (!disconnected) res.type('json').send(serialized); + } catch (error) { + if (!disconnected) + res.status(error.code === 'deadline' ? 504 : 502).json({ + error: + error.code === 'deadline' ? 'deadline_exceeded' : 'capture_failed', + }); + } finally { + stopped = true; + clearTimeout(timer); + req.off('aborted', disconnect); + res.off('close', disconnect); + // Initiate cleanup before releasing admission. Late allocation is also + // observed and immediately closed, even after this response has ended. + await close(); + if (!allocationPending) busy = false; + } + }; +} +module.exports = { createHandler }; diff --git a/deployments/company-scraper/company-handler.test.cjs b/deployments/company-scraper/company-handler.test.cjs new file mode 100644 index 000000000..94eeea796 --- /dev/null +++ b/deployments/company-scraper/company-handler.test.cjs @@ -0,0 +1,209 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const { createHandler } = require('./company-handler.cjs'); +const deferred = () => { + let resolve; + const promise = new Promise((r) => (resolve = r)); + return { promise, resolve }; +}; +const tick = () => new Promise((r) => setImmediate(r)); +function fixture(overrides = {}) { + let closed = 0, + created = 0; + const context = { + close: async () => { + closed++; + }, + newPage: async () => ({ url: () => 'https://www.example.com/about' }), + }; + const deps = { + secret: 'test-secret', + ready: () => true, + assertSafeTargetUrl: async () => {}, + createContext: async (_, ua, observe) => { + created++; + assert.equal(ua, 'ThreadplaneCompanyResearch/1.0'); + observe(context); + return { context, securityState: {} }; + }, + scrapePage: async (_page, _url, waitUntil) => { + assert.equal(waitUntil, 'domcontentloaded'); + return { content: 'company', status: 200 }; + }, + budgetMs: 25, + ...overrides, + }; + const handler = createHandler(deps); + const request = ( + body = { url: 'https://example.com/' }, + auth = 'Bearer test-secret', + aborted = false + ) => { + const req = new EventEmitter(); + req.body = body; + req.get = () => auth; + req.aborted = aborted; + const res = new EventEmitter(); + res.statusCode = 200; + res.status = (n) => { + res.statusCode = n; + return res; + }; + res.json = (value) => { + res.body = value; + res.writableEnded = true; + return res; + }; + res.type = () => res; + res.send = (value) => res.json(JSON.parse(value)); + return { req, res, done: handler(req, res) }; + }; + return { + request, + get closed() { + return closed; + }, + get created() { + return created; + }, + }; +} +test('secret mandatory, authentication and strict homepage input', async () => { + assert.throws(() => createHandler({ secret: '' }), /secret/i); + const f = fixture(); + for (const [body, auth, status] of [ + [{ url: 'https://example.com/' }, '', 401], + [{ url: 'https://example.com/', headers: {} }, undefined, 400], + [{ url: 'http://example.com/' }, undefined, 400], + [{ url: 'https://user@example.com/' }, undefined, 400], + [{ url: 'https://example.com/path' }, undefined, 400], + ]) { + const r = f.request(body, auth); + await r.done; + assert.equal(r.res.statusCode, status); + } + assert.equal(f.created, 0); +}); +test('returns actual final URL and closes browser context', async () => { + const f = fixture(); + const r = f.request(); + await r.done; + assert.deepEqual(r.res.body, { + content: 'company', + pageStatusCode: 200, + sourceURL: 'https://example.com/', + url: 'https://www.example.com/about', + }); + assert.equal(f.closed, 1); +}); +test('deadline covers DNS and does not start browser when DNS resolves late', async () => { + const dns = deferred(); + const f = fixture({ assertSafeTargetUrl: () => dns.promise }); + const r = f.request(); + await r.done; + assert.equal(r.res.statusCode, 504); + dns.resolve(); + await tick(); + assert.equal(f.created, 0); +}); +test('deadline closes a context created late and rejects concurrent work', async () => { + const pending = deferred(); + let closed = 0; + const f = fixture({ + createContext: async (_, ua, observe) => { + await pending.promise; + const context = { close: async () => closed++ }; + observe(context); + return { context }; + }, + }); + const first = f.request(); + const busy = f.request(); + await busy.done; + assert.equal(busy.res.statusCode, 503); + await first.done; + assert.equal(first.res.statusCode, 504); + const stillBusy = f.request(); + await stillBusy.done; + assert.equal(stillBusy.res.statusCode, 503); + pending.resolve(); + await tick(); + assert.equal(closed, 1); +}); +test('disconnect during navigation closes context without returning content', async () => { + const navigating = deferred(); + const f = fixture({ scrapePage: () => navigating.promise }); + const r = f.request(); + await tick(); + r.res.emit('close'); + await r.done; + assert.equal(f.closed, 1); + assert.equal(r.res.body, undefined); + navigating.resolve({ content: 'late', status: 200 }); +}); +test('deadline includes route setup after context allocation', async () => { + const setup = deferred(); + let closed = 0; + const f = fixture({ + createContext: async (_, ua, observe) => { + const context = { close: async () => closed++ }; + observe(context); + await setup.promise; + return { context }; + }, + }); + const r = f.request(); + await r.done; + assert.equal(r.res.statusCode, 504); + assert.equal(closed, 1); + setup.resolve(); +}); +test('rejects unsafe final URL and serialized response above cap', async () => { + const f = fixture({ + assertSafeTargetUrl: async (url) => { + if (url.includes('/about')) throw Error('private URL must not leak'); + }, + }); + const r = f.request(); + await r.done; + assert.equal(r.res.statusCode, 502); + assert.deepEqual(r.res.body, { error: 'capture_failed' }); + const big = fixture({ + scrapePage: async () => ({ content: '"'.repeat(1024 * 1024), status: 200 }), + }); + const b = big.request(); + await b.done; + assert.equal(b.res.statusCode, 502); +}); +test('rejects empty query/fragment delimiters and URLs longer than contract cap', async () => { + const f = fixture(); + for (const url of [ + 'https://example.com/?', + 'https://example.com/#', + `https://${'a'.repeat(490)}.com/`, + ]) { + const r = f.request({ url }); + await r.done; + assert.equal(r.res.statusCode, 400); + } +}); +test('already aborted requests never allocate context or write response', async () => { + const f = fixture(); + const r = f.request(undefined, undefined, true); + await r.done; + assert.equal(f.created, 0); + assert.equal(r.res.body, undefined); +}); +test('synchronous extraction cannot send after elapsed deadline', async () => { + const f = fixture({ + scrapePage: async () => { + const end = performance.now() + 35; + while (performance.now() < end) {} + return { content: 'late', status: 200 }; + }, + }); + const r = f.request(); + await r.done; + assert.equal(r.res.statusCode, 504); +}); diff --git a/deployments/company-scraper/project.json b/deployments/company-scraper/project.json new file mode 100644 index 000000000..608e3091a --- /dev/null +++ b/deployments/company-scraper/project.json @@ -0,0 +1,10 @@ +{ + "name": "company-scraper", + "tags": ["scope:growth-lifecycle", "type:app", "runtime:node22"], + "targets": { + "test": { + "executor": "nx:run-commands", + "options": { "command": "node --test deployments/company-scraper/*.test.cjs" } + } + } +} diff --git a/deployments/company-scraper/upstream.patch b/deployments/company-scraper/upstream.patch new file mode 100644 index 000000000..e0853b0a3 --- /dev/null +++ b/deployments/company-scraper/upstream.patch @@ -0,0 +1,437 @@ +--- a/apps/playwright-service-ts/api.ts ++++ b/apps/playwright-service-ts/api.ts +@@ -9,7 +9,7 @@ + } from 'playwright'; + import dotenv from 'dotenv'; + import UserAgent from 'user-agents'; +-import { getError } from './helpers/get_error'; ++const { createHandler } = require('./company-handler.cjs'); + import { lookup } from 'dns/promises'; + import IPAddr from 'ipaddr.js'; + import { Server, RequestError } from 'proxy-chain'; +@@ -19,20 +19,15 @@ + const app = express(); + const port = process.env.PORT || 3003; + +-app.use(express.json()); ++if (!process.env.COMPANY_SCRAPER_SECRET?.trim()) throw new Error('COMPANY_SCRAPER_SECRET is required'); ++app.use(express.json({ limit: '4kb' })); + + const BLOCK_MEDIA = + (process.env.BLOCK_MEDIA || 'False').toUpperCase() === 'TRUE'; +-const MAX_CONCURRENT_PAGES = Math.max( +- 1, +- Number.parseInt(process.env.MAX_CONCURRENT_PAGES ?? '10', 10) || 10, +-); +-const ALLOW_LOCAL_WEBHOOKS = +- (process.env.ALLOW_LOCAL_WEBHOOKS || 'False').toUpperCase() === 'TRUE'; +- +-const PROXY_SERVER = process.env.PROXY_SERVER || null; +-const PROXY_USERNAME = process.env.PROXY_USERNAME || null; +-const PROXY_PASSWORD = process.env.PROXY_PASSWORD || null; ++const ALLOW_LOCAL_WEBHOOKS = false; ++const PROXY_SERVER = null; ++const PROXY_USERNAME = null; ++const PROXY_PASSWORD = null; + + class InsecureConnectionError extends Error { + constructor( +@@ -83,17 +78,6 @@ + 'resolves to a private/internal address', + ); + } +-}; +- +-const buildUpstreamProxyUrl = (): string | undefined => { +- if (!PROXY_SERVER) return undefined; +- const server = PROXY_SERVER.includes('://') +- ? PROXY_SERVER +- : `http://${PROXY_SERVER}`; +- const url = new URL(server); +- if (PROXY_USERNAME) url.username = PROXY_USERNAME; +- if (PROXY_PASSWORD) url.password = PROXY_PASSWORD; +- return url.toString(); + }; + + const startSSRFProxy = async (): Promise => { +@@ -107,7 +91,7 @@ + 403, + ); + } +- return { upstreamProxyUrl: buildUpstreamProxyUrl() }; ++ return { upstreamProxyUrl: undefined }; + }, + }); + await server.listen(); +@@ -119,46 +103,6 @@ + type ContextSecurityState = { + blockedNavigationRequestUrl: string | null; + }; +-class Semaphore { +- private permits: number; +- private queue: (() => void)[] = []; +- +- constructor(permits: number) { +- this.permits = permits; +- } +- +- async acquire(): Promise { +- if (this.permits > 0) { +- this.permits--; +- return Promise.resolve(); +- } +- +- return new Promise((resolve) => { +- this.queue.push(resolve); +- }); +- } +- +- release(): void { +- this.permits++; +- if (this.queue.length > 0) { +- const nextResolve = this.queue.shift(); +- if (nextResolve) { +- this.permits--; +- nextResolve(); +- } +- } +- } +- +- getAvailablePermits(): number { +- return this.permits; +- } +- +- getQueueLength(): number { +- return this.queue.length; +- } +-} +-const pageSemaphore = new Semaphore(MAX_CONCURRENT_PAGES); +- + const AD_SERVING_DOMAINS = [ + 'doubleclick.net', + 'adservice.google.com', +@@ -175,15 +119,6 @@ + 'amazon-adsystem.com', + ]; + +-interface UrlModel { +- url: string; +- wait_after_load?: number; +- timeout?: number; +- headers?: { [key: string]: string }; +- check_selector?: string; +- skip_tls_verification?: boolean; +-} +- + let browser: Browser; + + const initializeBrowser = async () => { +@@ -204,6 +139,7 @@ + const createContext = async ( + skipTlsVerification: boolean = false, + userAgentOverride?: string, ++ onContext?: (context: BrowserContext) => void, + ): Promise<{ + context: BrowserContext; + securityState: ContextSecurityState; +@@ -226,6 +162,7 @@ + }; + + const newContext = await browser.newContext(contextOptions); ++ onContext?.(newContext); + + if (BLOCK_MEDIA) { + await newContext.route( +@@ -248,7 +185,7 @@ + if (request.isNavigationRequest()) { + securityState.blockedNavigationRequestUrl = requestUrlString; + } +- console.warn(`Blocked request: ${requestUrlString}`); ++ // Do not log target URLs. + return route.abort('blockedbyclient'); + } + throw error; +@@ -257,7 +194,7 @@ + const hostname = new URL(requestUrlString).hostname.toLowerCase(); + + if (AD_SERVING_DOMAINS.some((domain) => hostname.includes(domain))) { +- console.log(hostname); ++ + return route.abort(); + } + return route.continue(); +@@ -270,15 +207,6 @@ + const shutdownBrowser = async () => { + if (browser) { + await browser.close(); +- } +-}; +- +-const isValidUrl = (urlString: string): boolean => { +- try { +- new URL(urlString); +- return true; +- } catch (_) { +- return false; + } + }; + +@@ -285,15 +213,12 @@ + const scrapePage = async ( + page: Page, + url: string, +- waitUntil: 'load' | 'networkidle', ++ waitUntil: 'load' | 'networkidle' | 'domcontentloaded', + waitAfterLoad: number, + timeout: number, + checkSelector: string | undefined, + securityState: ContextSecurityState, + ) => { +- console.log( +- `Navigating to ${url} with waitUntil: ${waitUntil} and timeout: ${timeout}ms`, +- ); + let response; + try { + response = await page.goto(url, { waitUntil, timeout }); +@@ -344,218 +269,22 @@ + }; + }; + +-app.get('/health', async (req: Request, res: Response) => { +- try { +- if (!browser) { +- await initializeBrowser(); +- } +- +- const { context: testContext } = await createContext(); +- const testPage = await testContext.newPage(); +- await testPage.close(); +- await testContext.close(); +- +- res.status(200).json({ +- status: 'healthy', +- maxConcurrentPages: MAX_CONCURRENT_PAGES, +- activePages: MAX_CONCURRENT_PAGES - pageSemaphore.getAvailablePermits(), +- }); +- } catch (error) { +- console.error('Health check failed:', error); +- res.status(503).json({ +- status: 'unhealthy', +- error: error instanceof Error ? error.message : 'Unknown error occurred', +- }); +- } ++app.get('/health', (_req: Request, res: Response) => { ++ const ready = Boolean(browser?.isConnected()); ++ res.status(ready ? 200 : 503).json({ status: ready ? 'ready' : 'unavailable' }); + }); + +-app.post('/scrape', async (req: Request, res: Response) => { +- const { +- url, +- wait_after_load = 0, +- timeout = 15000, +- headers, +- check_selector, +- skip_tls_verification = false, +- }: UrlModel = req.body; +- +- console.log(`================= Scrape Request =================`); +- console.log(`URL: ${url}`); +- console.log(`Wait After Load: ${wait_after_load}`); +- console.log(`Timeout: ${timeout}`); +- console.log(`Headers: ${headers ? JSON.stringify(headers) : 'None'}`); +- console.log(`Check Selector: ${check_selector ? check_selector : 'None'}`); +- console.log(`Skip TLS Verification: ${skip_tls_verification}`); +- console.log(`==================================================`); +- +- if (!url) { +- return res.status(400).json({ error: 'URL is required' }); +- } +- +- if (!isValidUrl(url)) { +- return res.status(400).json({ error: 'Invalid URL' }); +- } +- +- try { +- await assertSafeTargetUrl(url); +- } catch (error) { +- if (error instanceof InsecureConnectionError) { +- return res.json({ +- content: '', +- pageStatusCode: 403, +- pageError: error.message, +- }); +- } +- throw error; +- } +- +- if (!PROXY_SERVER) { +- console.warn( +- '⚠️ WARNING: No proxy server provided. Your IP address may be blocked.', +- ); +- } +- +- if (!browser) { +- await initializeBrowser(); +- } +- +- await pageSemaphore.acquire(); +- +- let requestContext: BrowserContext | null = null; +- let securityState: ContextSecurityState | null = null; +- let page: Page | null = null; +- +- try { +- // Extract user-agent from request headers (case-insensitive) so it can +- // be applied at the context level. Playwright ignores user-agent in +- // setExtraHTTPHeaders when the context already defines one (#2802). +- const userAgentOverride = headers +- ? Object.entries(headers).find( +- ([k]) => k.toLowerCase() === 'user-agent', +- )?.[1] +- : undefined; +- +- const contextBundle = await createContext( +- skip_tls_verification, +- userAgentOverride, +- ); +- requestContext = contextBundle.context; +- securityState = contextBundle.securityState; +- page = await requestContext.newPage(); +- +- if (headers) { +- // A Cookie header passed through setExtraHTTPHeaders is sent on the first +- // request but DROPPED on any redirect hop (the browser regenerates the +- // redirected request from its cookie jar, which is empty). Authenticated +- // sites that 302 (e.g. to /signin when the session looks absent) then +- // land on the login page. Seed the cookie jar instead so Chromium re-sends +- // it on every request, including redirects — matching what a raw HTTP +- // client does. +- const cookieHeader = Object.entries(headers).find( +- ([k]) => k.toLowerCase() === 'cookie', +- )?.[1]; +- if (cookieHeader) { +- // Scope cookies to the registrable domain (e.g. ".example.com"), not +- // host-only. Authenticated pages often 302 across sibling subdomains +- // (example.com -> app.example.com); a host-only cookie set for the +- // original host would not be sent to the redirect target, leaving the +- // request unauthenticated. The Cookie header carries no domain info, so +- // we apply the eTLD+1 — broad enough to follow the redirect, and these +- // are first-party cookies being returned to their own origin anyway. +- let cookieDomain: string | undefined; +- try { +- const host = new URL(url).hostname; +- const labels = host.split('.'); +- cookieDomain = labels.length > 2 ? labels.slice(-2).join('.') : host; +- } catch { +- cookieDomain = undefined; +- } +- type SeedCookie = { +- name: string; +- value: string; +- url?: string; +- domain?: string; +- path?: string; +- }; +- const cookies = cookieHeader +- .split(';') +- .map((pair) => pair.trim()) +- .filter(Boolean) +- .map((pair): SeedCookie | null => { +- const eq = pair.indexOf('='); +- if (eq === -1) return null; +- const name = pair.slice(0, eq).trim(); +- const value = pair.slice(eq + 1).trim(); +- return cookieDomain +- ? { name, value, domain: `.${cookieDomain}`, path: '/' } +- : { name, value, url }; +- }) +- .filter((c): c is SeedCookie => c !== null); +- if (cookies.length > 0) { +- try { +- await requestContext.addCookies(cookies); +- } catch (error) { +- console.warn('Failed to seed cookies from Cookie header:', error); +- } +- } +- } +- +- // Remove user-agent (already applied at the context level) and cookie +- // (now seeded into the jar) before forwarding the rest verbatim. +- const filteredHeaders = Object.fromEntries( +- Object.entries(headers).filter(([k]) => { +- const lower = k.toLowerCase(); +- return lower !== 'user-agent' && lower !== 'cookie'; +- }), +- ); +- if (Object.keys(filteredHeaders).length > 0) { +- await page.setExtraHTTPHeaders(filteredHeaders); +- } +- } +- +- const result = await scrapePage( +- page, +- url, +- 'load', +- wait_after_load, +- timeout, +- check_selector, +- securityState, +- ); +- const pageError = +- result.status !== 200 ? getError(result.status) : undefined; +- +- if (!pageError) { +- console.log(`✅ Scrape successful!`); +- } else { +- console.log( +- `🚨 Scrape failed with status code: ${result.status} ${pageError}`, +- ); +- } +- +- res.json({ +- content: result.content, +- pageStatusCode: result.status, +- contentType: result.contentType, +- ...(pageError && { pageError }), +- }); +- } catch (error) { +- if (error instanceof InsecureConnectionError) { +- return res.json({ +- content: '', +- pageStatusCode: 403, +- pageError: error.message, +- }); +- } +- console.error('Scrape error:', error); +- res +- .status(500) +- .json({ error: 'An error occurred while fetching the page.' }); +- } finally { +- if (page) await page.close(); +- if (requestContext) await requestContext.close(); +- pageSemaphore.release(); +- } ++app.post('/scrape', createHandler({ ++ secret: process.env.COMPANY_SCRAPER_SECRET, ++ ready: () => Boolean(browser?.isConnected()), ++ assertSafeTargetUrl, ++ createContext, ++ scrapePage, ++})); ++ ++// Body parser errors must not echo request bodies or framework stack traces. ++app.use((_error: unknown, _req: Request, res: Response, _next: unknown) => { ++ res.status(400).json({ error: 'invalid_request' }); + }); + + const start = async () => { +@@ -566,7 +295,7 @@ + }); + }; + start().catch((error) => { +- console.error('Failed to start server:', error); ++ console.error('Failed to start scraper'); + process.exit(1); + }); +