feat(sim): share the simulation runner with the host repos

.github/workflows/reuse-quality.yml

@@ -40,6 +40,9 @@ jobs:
     - name: Start dev API
       run: dotenv -- npm run dev-api &
 
+    - name: Install Playwright browser
+      run: npx playwright install --with-deps chromium
+
     - name: Run unit tests
       run: npm run test-unit
 

lib-sim/README.md

@@ -2,9 +2,10 @@
 
 Primitives for judged browser simulations of an [agents](https://github.com/data-fair/agents)
 chat: drive a real conversation with a simulated user in a real browser, capture
-what happened, and hand the transcript to a judge. It does not own cases, login,
-settings seeding, or the turn loop — those stay in the host repo, which knows
-its own routes, fixtures and account setup. It also ships a small Claude Code
+what happened, and hand the transcript to a judge. It owns the turn loop
+(`runSimulation`) and the agents side of the setup; cases, login and the host's
+own seeding stay in the host repo, which knows its own routes, fixtures and
+account setup. It also ships a small Claude Code
 bridge (a dev tool, not a simulation primitive) that exposes the Claude Agent
 SDK as an OpenAI-compatible provider, so a dev workspace can drive the assistant
 under test on a Claude subscription instead of a metered API key.
@@ -60,8 +61,65 @@ The command prints the package version it copied from so drift is at least
 detectable; without `--force` it skips a file that already exists rather than
 overwriting local edits.
 
+## Running cases in a data-fair host
+
+What data-fair, portals and catalogs share: the turn loop and its validity
+guards (`runSimulation`), the bridge settings of the simulation owner
+(`seedAgentsSettings`), the chat drawer helpers (`CHAT_FRAME`, `ensureChatOpen`),
+the Playwright config (`defineSimConfig`) and the report script (`reportCli`).
+
+```ts
+// playwright.sim.config.ts
+import 'dotenv/config'
+import { defineSimConfig } from '@data-fair/lib-agents-sim'
+
+export default defineSimConfig({ baseURL: `http://${process.env.DEV_HOST}:${process.env.NGINX_PORT}` })
+```
+
+```ts
+// simulations/simulate.sim.spec.ts
+import { test } from '@playwright/test'
+import { runSimulation, seedAgentsSettings, selectCases, simCaseNames } from '@data-fair/lib-agents-sim'
+import { cases } from './cases/index.ts'
+
+for (const simCase of selectCases(cases, simCaseNames())) {
+  test(`simulation: ${simCase.name}`, async ({ page }) => {
+    await runSimulation(page, simCase, {
+      agentsUrl: `${ROOT}/agents`,
+      locale: 'fr',
+      // inside the run: a failure here is recorded as an invalid run
+      setup: async ({ assistantModel, toolsModel }) => {
+        await seedYourData()
+        await seedAgentsSettings(superadminAxios, `${ROOT}/agents`, OWNER, assistantModel, toolsModel)
+        await turnOnYourChat()
+        await loginAndGoTo(page, simCase.route)
+      },
+      perceptionRoots: [{ label: 'page', root: page, frames: true, cap: 16000 }]
+    })
+  })
+}
+```
+
+```ts
+// simulations/report.ts
+import { reportCli } from '@data-fair/lib-agents-sim'
+import { cases } from './cases/index.ts'
+
+reportCli(cases)
+```
+
+`runSimulation` reads `SIM_ASSISTANT_MODEL` (default `sonnet`), `SIM_TOOLS_MODEL`
+(default `haiku`, the background roles), `SIM_USER_MODEL` and `SIM_SUB_AGENTS=0`,
+and checks the bridge (`BRIDGE_URL`, or `BRIDGE_PORT`) before `setup`. It writes a
+pessimistic sidecar first, hands a `wait_for_user_action` over to the person,
+puts the actions of a silent pass back in the conversation, accepts dialogs (and
+says so in `consoleErrors`), and records a provider error, a cold Vite dev server,
+an empty transcript or no gateway exchange as an invalid run, then throws.
+
 ## Minimal runner example
 
+To write your own loop instead:
+
 A host repo owns the Playwright test that drives one case end to end. For a
 chat embedded in an iframe (the common case for a host application), pass
 `page.frameLocator('iframe')` to `createChatDriver`; for a page where the chat

lib-sim/bridge-settings.ts

@@ -0,0 +1,100 @@
+/**
+ * Point a simulation owner's agent settings at the Claude Code bridge, so the
+ * assistant under test runs on a real model instead of the dev mock provider.
+ */
+
+export const BRIDGE_URL = process.env.BRIDGE_URL ?? `http://localhost:${process.env.BRIDGE_PORT ?? 3194}/v1`
+
+export const MODEL_ROLES = ['assistant', 'tools', 'summarizer', 'evaluator', 'moderator'] as const
+export type ModelRole = typeof MODEL_ROLES[number]
+
+/**
+ * Roles a deployment puts on a small model: sub-agents, compaction, the
+ * moderation guard. Running them on the assistant's model would both cost more
+ * per case and flatter the product — a sub-agent prompt that only a large model
+ * can follow reads as working until a real deployment runs it on the cheap tier.
+ * The evaluator is a trace-review role no run exercises, so it follows the
+ * assistant rather than earning a third setting.
+ */
+export const BACKGROUND_ROLES: readonly ModelRole[] = ['tools', 'summarizer', 'moderator']
+
+export type SimOwner = { type: string, id: string }
+
+/** The subset of an axios instance the seeding uses. */
+export type HttpClient = {
+  put: (url: string, body: unknown) => Promise<unknown>
+  post: (url: string, body: unknown) => Promise<unknown>
+}
+
+const unlimited = { unlimited: true, monthlyLimit: 0 }
+/** Every role unlimited: portal cases are anonymous visitors, back-office cases are admins. */
+const quotas = { admin: unlimited, contrib: unlimited, user: unlimited, external: unlimited, anonymous: unlimited, untrusted: unlimited }
+
+/**
+ * The two bodies the agents API takes since settings were split by author:
+ * `superadmin` (the provider and the model catalog, each model listing the
+ * roles it may serve) and `org` (which model each role uses, quotas, traces).
+ * The org body is validated against the catalog, so it is written second.
+ */
+export function bridgeSettings (assistantModelId: string, toolsModelId: string, bridgeUrl = BRIDGE_URL) {
+  const bridge = { type: 'openai-compatible', id: 'bridge', name: 'Claude Code Bridge' }
+  const modelFor = (role: ModelRole) => BACKGROUND_ROLES.includes(role) ? toolsModelId : assistantModelId
+  const ids = [...new Set(MODEL_ROLES.map(modelFor))]
+  return {
+    superadmin: {
+      providers: [{
+        ...bridge,
+        enabled: true,
+        baseURL: bridgeUrl,
+        // MANDATORY: in 'default' mode the provider targets /v1/responses, which the bridge does not implement.
+        compatibility: 'compatible'
+      }],
+      models: ids.map(id => ({
+        model: { id, name: id, provider: bridge },
+        usage: MODEL_ROLES.filter(role => modelFor(role) === id),
+        inputPricePerMillion: 0,
+        outputPricePerMillion: 0
+      }))
+    },
+    org: {
+      modelMapping: Object.fromEntries(
+        MODEL_ROLES.map(role => [role, { provider: bridge.id, id: modelFor(role), name: modelFor(role) }])
+      ) as Record<ModelRole, { provider: string, id: string, name: string }>,
+      quotas,
+      storeTraces: false
+    }
+  }
+}
+
+/**
+ * Write the agents settings of `owner` (superadmin body, then org body) and lift
+ * its credit cap. `admin` is a superadmin client in admin mode, `agentsUrl` the
+ * agents service's base URL (`<origin>/agents` behind the dev nginx). Turning the
+ * chat on in the host service stays with the host.
+ */
+export async function seedAgentsSettings (admin: HttpClient, agentsUrl: string, owner: SimOwner, assistantModelId: string, toolsModelId: string) {
+  const { superadmin, org } = bridgeSettings(assistantModelId, toolsModelId)
+  await admin.put(`${agentsUrl}/api/settings/${owner.type}/${owner.id}`, superadmin)
+  await admin.put(`${agentsUrl}/api/settings/${owner.type}/${owner.id}/org`, org)
+  // The agents service caps each account's AI credits before any quota, and a
+  // production-mode image defaults that cap to 0: every request is refused with a
+  // 429. -1 lifts it; consumption is reset so a run never inherits the last.
+  await admin.post(`${agentsUrl}/api/v1/limits/${owner.type}/${owner.id}`, {
+    lastUpdate: new Date().toISOString(),
+    ai_credits: { limit: -1, consumption: 0 }
+  })
+}
+
+/** Fail loudly and early: without the bridge every case dies as an opaque timeout. */
+export async function assertBridgeUp (bridgeUrl = BRIDGE_URL) {
+  const statusUrl = bridgeUrl.replace(/\/v1$/, '') + '/_bridge/status'
+  try {
+    const res = await fetch(statusUrl, { signal: AbortSignal.timeout(3000) })
+    if (!res.ok) throw new Error(`HTTP ${res.status}`)
+  } catch (err) {
+    throw new Error(
+      `The Claude Code bridge is not answering at ${statusUrl} (${err instanceof Error ? err.message : String(err)}).\n` +
+      'Ask your user to start it with: npm run dev-bridge'
+    )
+  }
+}

lib-sim/cases.ts

@@ -16,3 +16,8 @@ export function selectCases <T extends SimulationCase> (all: T[], names: string[
     return found
   })
 }
+
+/** Case names from SIM_CASES (comma-separated), empty for all. */
+export function simCaseNames (env: { SIM_CASES?: string } = process.env): string[] {
+  return (env.SIM_CASES ?? '').split(',').map(s => s.trim()).filter(Boolean)
+}

lib-sim/gateway-errors.ts

@@ -0,0 +1,82 @@
+/**
+ * Tell a provider failure apart from a product failure.
+ *
+ * When the model stream dies — a rate limit, a dead bridge, an unknown model —
+ * the agents gateway does not fail the HTTP request. It writes an error chunk
+ * into the SSE body and closes cleanly (api/src/gateway/router.ts). The chat
+ * then renders that as an alert, the Stop button goes away, `waitForTurn`
+ * returns normally and nothing throws. The run looks valid, the judge reads a
+ * transcript of an assistant that never answered, and reports the product as
+ * unsatisfactory for something the harness did to it.
+ *
+ * So we read the gateway's own protocol instead of the screen: the `error`
+ * field of a chunk, which is a JSON shape the service emits, not prose. It
+ * survives translation, restyling and any rewording of the message itself —
+ * unlike matching on what the alert says, which is why that approach was
+ * rejected.
+ */
+import type { Page, Response } from '@playwright/test'
+
+export type GatewayErrors = {
+  /** Messages from every error chunk seen, in arrival order. */
+  messages: string[]
+  /** Await the in-flight body reads before deciding a run's validity. */
+  settle: () => Promise<void>
+}
+
+/** Pull the error messages out of one SSE body, ignoring everything else. */
+export function errorsInSseBody (body: string): string[] {
+  const found: string[] = []
+  for (const line of body.split('\n')) {
+    if (!line.startsWith('data:')) continue
+    const payload = line.slice(5).trim()
+    if (!payload || payload === '[DONE]') continue
+    try {
+      const message = JSON.parse(payload)?.error?.message
+      if (typeof message === 'string' && message) found.push(message)
+    } catch {
+      // A chunk that is not JSON is a normal content delta, not a signal.
+    }
+  }
+  return found
+}
+
+/**
+ * Errors in one gateway response. A quota refusal (the account credit cap,
+ * checked before any stream opens) is an HTTP 429 with a JSON body, not an SSE
+ * error chunk — the chat retries it and renders the same alert, so it must
+ * invalidate the run just the same. Authentication refusals are the exception
+ * (see below).
+ */
+export function errorsInResponse (status: number, body: string): string[] {
+  if (status < 400) return errorsInSseBody(body)
+  // 401/403 are the agents service refusing this visitor, not the model failing:
+  // a portal case's anonymous visitor once had its first message refused because
+  // its action token was "not yet valid". The person sees that refusal in the chat,
+  // so it is product behaviour for the judge, not a reason to throw the run away.
+  if (status === 401 || status === 403) return []
+  let reason: unknown
+  try { reason = JSON.parse(body)?.reason ?? JSON.parse(body)?.error?.message } catch {}
+  return [`HTTP ${status}${typeof reason === 'string' && reason ? `: ${reason}` : ''}`]
+}
+
+export function captureGatewayErrors (page: Page): GatewayErrors {
+  const messages: string[] = []
+  const pending: Promise<unknown>[] = []
+
+  page.on('response', (res: Response) => {
+    if (!res.url().includes('/v1/chat/completions')) return
+    // Bodies are read off the event loop and awaited later: reading here would
+    // block the handler on a stream that has not finished, and a listener that
+    // never returns stalls Playwright's event dispatch.
+    pending.push(
+      res.text()
+        .then(body => { messages.push(...errorsInResponse(res.status(), body)) })
+        // An unreadable body (navigated away, aborted) is not evidence of an
+        // error — only a parsed error chunk is.
+        .catch(() => {})
+    )
+  })
+
+  return { messages, settle: async () => { await Promise.allSettled(pending) } }
+}

lib-sim/index.ts

@@ -1,11 +1,16 @@
 export type { SimulationCase, Transcript, RunSidecar } from './types.ts'
 export { createNeutralCwd, scrubEnv, isolationOptions, type Env } from './isolation.ts'
 export { captureGateway, summariseRequest, type GatewayExchange } from './gateway-capture.ts'
+export { captureGatewayErrors, errorsInSseBody, errorsInResponse, type GatewayErrors } from './gateway-errors.ts'
 export { nextUserMessage, personaSystemPrompt, personaPrompt, isDone, DONE, PERSONA_MAX_TURNS, PERCEPTION_INSTRUCTIONS, DEFAULT_USER_MODEL, resolveUserModel } from './persona.ts'
 export { writeEvidence, evidenceDir } from './transcript.ts'
 export { computeMetrics, type RunMetrics } from './metrics.ts'
-export { selectCases } from './cases.ts'
-export { reportCases } from './report.ts'
+export { selectCases, simCaseNames } from './cases.ts'
+export { reportCases, reportCli } from './report.ts'
 export { createChatDriver, chatDriverStrings, type ChatRoot, type ChatDriverLocale, type TurnOutcome, TURN_TIMEOUT_MS, SEND_TIMEOUT_MS, WAITING_SELECTOR } from './chat-driver.ts'
 export { createPagePerception, truncate, SNAPSHOT_CAP, ACTION_TIMEOUT_MS, MCP_SERVER_NAME as PAGE_MCP_SERVER_NAME } from './page-perception.ts'
 export type { PerceptionRoot, PerceptionContent, Observation, PagePerception } from './page-perception.ts'
+export { passActions, withSilentPasses, type SilentPass } from './silent-passes.ts'
+export { BRIDGE_URL, MODEL_ROLES, BACKGROUND_ROLES, bridgeSettings, seedAgentsSettings, assertBridgeUp, type ModelRole, type SimOwner, type HttpClient } from './bridge-settings.ts'
+export { runSimulation, ensureChatOpen, CHAT_FRAME, TURN_CEILING_MS, type SimulationOptions, type SimulationModels } from './run-simulation.ts'
+export { defineSimConfig, type SimConfigOptions } from './sim-config.ts'

lib-sim/report.ts

@@ -5,6 +5,8 @@
 import fs from 'node:fs'
 import path from 'node:path'
 import type { SimulationCase, RunSidecar } from './types.ts'
+import { selectCases, simCaseNames } from './cases.ts'
+import { evidenceDir } from './transcript.ts'
 
 type Verdict = {
   case: string
@@ -91,3 +93,14 @@ export function reportCases (cases: SimulationCase[], evidenceDir: string): numb
   console.log(failures === 0 ? '\nall cases satisfied' : `\n${failures} case(s) need attention`)
   return failures
 }
+
+/**
+ * A host's `simulate:report` script. argv wins, then SIM_CASES, so a report after
+ * a filtered `SIM_CASES=… npm run simulate` reports on the cases that actually ran
+ * instead of failing on the ones that deliberately did not.
+ */
+export function reportCli (cases: SimulationCase[], argv = process.argv.slice(2)) {
+  // exitCode, not process.exit(): stdout is async when piped (it is, under npm
+  // run) and process.exit discards pending writes, truncating the report.
+  process.exitCode = reportCases(selectCases(cases, argv.length ? argv : simCaseNames()), evidenceDir) === 0 ? 0 : 1
+}