crawler: read Obscura replies as SSE by content type (0.49.1)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pitworld",
-  "version": "0.49.0",
+  "version": "0.49.1",
   "mcpName": "world.pit/pit",
   "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.",
   "license": "MIT",

services/crawler/src/obscura.ts

@@ -69,6 +69,18 @@ export function documentFrom(parts: { lang?: string; head?: string; text?: strin
   return `<!doctype html><html lang="${esc(parts.lang ?? '')}"><head>${head}</head><body>${main}${rest}<nav>${anchors}</nav></body></html>`
 }
 
+/**
+ * The JSON-RPC message in a reply: an SSE stream's last `data:` line, or the body as is.
+ * Decided by the content type, never by "data:" appearing in the text: a plain JSON reply
+ * holding a page head with a data: URI was read as SSE, found no data line, and threw.
+ */
+export function sseData(text: string, contentType: string | null): string {
+  const sse = /text\/event-stream/i.test(contentType ?? '') || /^\s*(event|data|id):/m.test(text.slice(0, 200)) && !text.trimStart().startsWith('{')
+  if (!sse) return text
+  const data = text.split('\n').filter((l) => l.startsWith('data:'))
+  return data.length ? data[data.length - 1]!.slice(5) : text
+}
+
 export class ObscuraClient {
   private session = ''
   private id = 0
@@ -120,13 +132,7 @@ export class ObscuraClient {
     }
     if (!response.ok) throw new ObscuraError(`obscura HTTP ${response.status}`)
     if (notify) return null
-    const line = text.includes('data:')
-      ? text
-          .split('\n')
-          .filter((l) => l.startsWith('data:'))
-          .pop()!
-          .slice(5)
-      : text
+    const line = sseData(text, response.headers.get('content-type'))
     let json: any
     try {
       json = JSON.parse(line)

services/crawler/test/failures.test.ts

@@ -8,7 +8,7 @@ import { createCrawler, type PageJob, type SiteJob } from '../src/crawler.ts'
 import { failureKind, HOST_PAGE_FAILURES, isDue, parkFor, retryAfterMs, statusKind } from '../src/failures.ts'
 import { politeGet } from '../src/fetcher.ts'
 import { MemoryFrontier } from '../src/frontier.ts'
-import { ObscuraClient, ObscuraError, RenderTimeout, SETTLED_EXPRESSION } from '../src/obscura.ts'
+import { ObscuraClient, ObscuraError, RenderTimeout, SETTLED_EXPRESSION, sseData } from '../src/obscura.ts'
 import { MemorySiteStore } from '../src/site-store.ts'
 
 const DAY = 86_400_000
@@ -285,3 +285,11 @@ describe('the browser budget', () => {
     expect((await frontier.siteState('js.example', 'https://js.example/')).pageDown).toBe(true)
   })
 })
+
+test('a JSON reply that mentions data: is JSON; an SSE reply is read by its last data line', () => {
+  const json = '{"jsonrpc":"2.0","id":3,"result":{"content":[{"text":"<link href=\\"data:image/png;base64,AA\\">"}]}}'
+  expect(sseData(json, 'application/json')).toBe(json)
+  expect(sseData(json, null)).toBe(json)
+  expect(sseData('event: message\ndata: {"a":1}\n\n', 'text/event-stream')).toBe(' {"a":1}')
+  expect(sseData('event: message\ndata: {"a":1}\n\n', null)).toBe(' {"a":1}')
+})