diff --git a/package.json b/package.json index b072383..4442789 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pitworld", - "version": "0.49.0", + "version": "0.49.1", "mcpName": "world.pit/pit", "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.", "license": "MIT", diff --git a/services/crawler/src/obscura.ts b/services/crawler/src/obscura.ts index 38fbeed..cfa5597 100644 --- a/services/crawler/src/obscura.ts +++ b/services/crawler/src/obscura.ts @@ -69,6 +69,18 @@ export function documentFrom(parts: { lang?: string; head?: string; text?: strin return `
${head}${main}${rest}` } +/** + * The JSON-RPC message in a reply: an SSE stream's last `data:` line, or the body as is. + * Decided by the content type, never by "data:" appearing in the text: a plain JSON reply + * holding a page head with a data: URI was read as SSE, found no data line, and threw. + */ +export function sseData(text: string, contentType: string | null): string { + const sse = /text\/event-stream/i.test(contentType ?? '') || /^\s*(event|data|id):/m.test(text.slice(0, 200)) && !text.trimStart().startsWith('{') + if (!sse) return text + const data = text.split('\n').filter((l) => l.startsWith('data:')) + return data.length ? data[data.length - 1]!.slice(5) : text +} + export class ObscuraClient { private session = '' private id = 0 @@ -120,13 +132,7 @@ export class ObscuraClient { } if (!response.ok) throw new ObscuraError(`obscura HTTP ${response.status}`) if (notify) return null - const line = text.includes('data:') - ? text - .split('\n') - .filter((l) => l.startsWith('data:')) - .pop()! - .slice(5) - : text + const line = sseData(text, response.headers.get('content-type')) let json: any try { json = JSON.parse(line) diff --git a/services/crawler/test/failures.test.ts b/services/crawler/test/failures.test.ts index 57cc390..8cd7ebb 100644 --- a/services/crawler/test/failures.test.ts +++ b/services/crawler/test/failures.test.ts @@ -8,7 +8,7 @@ import { createCrawler, type PageJob, type SiteJob } from '../src/crawler.ts' import { failureKind, HOST_PAGE_FAILURES, isDue, parkFor, retryAfterMs, statusKind } from '../src/failures.ts' import { politeGet } from '../src/fetcher.ts' import { MemoryFrontier } from '../src/frontier.ts' -import { ObscuraClient, ObscuraError, RenderTimeout, SETTLED_EXPRESSION } from '../src/obscura.ts' +import { ObscuraClient, ObscuraError, RenderTimeout, SETTLED_EXPRESSION, sseData } from '../src/obscura.ts' import { MemorySiteStore } from '../src/site-store.ts' const DAY = 86_400_000 @@ -285,3 +285,11 @@ describe('the browser budget', () => { expect((await frontier.siteState('js.example', 'https://js.example/')).pageDown).toBe(true) }) }) + +test('a JSON reply that mentions data: is JSON; an SSE reply is read by its last data line', () => { + const json = '{"jsonrpc":"2.0","id":3,"result":{"content":[{"text":""}]}}' + expect(sseData(json, 'application/json')).toBe(json) + expect(sseData(json, null)).toBe(json) + expect(sseData('event: message\ndata: {"a":1}\n\n', 'text/event-stream')).toBe(' {"a":1}') + expect(sseData('event: message\ndata: {"a":1}\n\n', null)).toBe(' {"a":1}') +})