From d5ba28767bc9013651305d22032601ec9b7dbc55 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Tue, 8 Sep 2026 07:59:14 -0700 Subject: [PATCH 1/2] test(cockpit-chat-debug): aimock e2e suite for the c-debug devtools dock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The c-debug demo had no e2e because it used to mount `` alone — a devtools dock with no composer, so nothing could produce a run and the Timeline tab showed its empty state forever. The demo now mounts `` beside the dock, so the capability is drivable and gets the same aimock-replay suite every sibling chat cap has. Four specs, wired the way the siblings wire theirs (playwright.config.ts + global-setup-impl.ts + fixtures/, and an `e2e` target in project.json, which is also what puts the cap in the CI matrix — scripts/cockpit-matrix.mjs derives that from `targets.e2e`): - the Timeline tab shows its empty state before any run; - a run sent through the composer fills it with the pipeline's checkpoint rows (asserted as the exact list, newest first, since the row set is precisely what this capability exists to demonstrate); - selecting a checkpoint diffs that step of the run; - the State tab swaps in the live state inspector. No assertion reads model prose. Every checkpoint label is structural — `toDebugCheckpoint` reads `state.next[0]`, so the rows are the graph wiring read backwards. The fixture is three entries, one per LLM call the graph makes in a turn (`generate`, `summarize`, `generate_title`), discriminated by `systemMessage` because all three carry the same user message. No `hasToolResult` ordering constraint applies: the graph binds no tools. Mutation-checked by removing `` from the demo template: the three run-driven specs fail on the missing composer and only the empty-state spec passes, which is exactly the state the deleted e2e/README.md described. Co-Authored-By: Claude Opus 5 --- cockpit/chat/debug/angular/e2e/README.md | 14 --- .../chat/debug/angular/e2e/c-debug.spec.ts | 99 +++++++++++++++++++ .../debug/angular/e2e/fixtures/c-debug.json | 31 ++++++ .../debug/angular/e2e/global-setup-impl.ts | 15 +++ .../debug/angular/e2e/playwright.config.ts | 20 ++++ cockpit/chat/debug/angular/e2e/tsconfig.json | 32 ++++++ cockpit/chat/debug/angular/project.json | 6 ++ 7 files changed, 203 insertions(+), 14 deletions(-) delete mode 100644 cockpit/chat/debug/angular/e2e/README.md create mode 100644 cockpit/chat/debug/angular/e2e/c-debug.spec.ts create mode 100644 cockpit/chat/debug/angular/e2e/fixtures/c-debug.json create mode 100644 cockpit/chat/debug/angular/e2e/global-setup-impl.ts create mode 100644 cockpit/chat/debug/angular/e2e/playwright.config.ts create mode 100644 cockpit/chat/debug/angular/e2e/tsconfig.json diff --git a/cockpit/chat/debug/angular/e2e/README.md b/cockpit/chat/debug/angular/e2e/README.md deleted file mode 100644 index bdc97d40a..000000000 --- a/cockpit/chat/debug/angular/e2e/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# c-debug — aimock e2e: not applicable - -The `c-debug` demo ships **without** an aimock-driven Playwright e2e suite. It now composes `` alongside ``, so the demo is drivable in principle: - -```html - - - - -``` - -What is still missing is a recorded fixture for this capability. Adding aimock coverage here means recording a first turn against the `c-debug` graph and asserting on the dock's timeline rows once it is opened from the fixed launcher. Track that as its own task. - -For now, only the manual smoke at `manual/debug.manual.ts` exercises this cap. diff --git a/cockpit/chat/debug/angular/e2e/c-debug.spec.ts b/cockpit/chat/debug/angular/e2e/c-debug.spec.ts new file mode 100644 index 000000000..69662d4ed --- /dev/null +++ b/cockpit/chat/debug/angular/e2e/c-debug.spec.ts @@ -0,0 +1,99 @@ +import { test, expect, type Page } from '@playwright/test'; +import { submitAndWaitForResponse } from '@threadplane-internal/e2e-harness'; + +const PROMPT = 'What is a jet bridge?'; + +/** + * The checkpoint rows the c-debug pipeline writes, newest first. + * + * Every label is structural, not model prose: `toDebugCheckpoint` reads + * `state.next[0]` off each LangGraph checkpoint, so the list is exactly the + * graph's wiring (`__start__` → generate → process → summarize → + * generate_title) read backwards, plus the terminal checkpoint whose `next` + * is empty and therefore falls back to the positional `Step 1` label. + * + * Asserting the whole list rather than a count is deliberate. The row set IS + * what this capability exists to demonstrate, so a node added to or dropped + * from the pipeline should fail here and be re-stated, not silently pass. + */ +const EXPECTED_CHECKPOINT_ROWS = [ + 'Step 1', + 'generate_title', + 'summarize', + 'process', + 'generate', + '__start__', +]; + +const openDock = (page: Page) => + page.getByRole('button', { name: /open chat devtools/i }).click(); + +test('c-debug: the Timeline tab shows its empty state before any run', async ({ page }) => { + await page.goto('/'); + await openDock(page); + + // The dock opens on the Timeline tab. With no run on the thread there is + // nothing to inspect, and this is the state the demo was stuck in for as + // long as it mounted with no composer beside it. + await expect(page.locator('chat-debug-timeline-inspector')).toBeVisible(); + await expect(page.getByText('No checkpoints yet.')).toBeVisible(); + await expect(page.locator('chat-debug-checkpoint-card')).toHaveCount(0); +}); + +test('c-debug: a run through the composer fills the Timeline tab', async ({ page }) => { + // Sending through 's composer is the whole point of the pairing: + // the chat produces the run, the dock inspects it. + await submitAndWaitForResponse(page, PROMPT); + await openDock(page); + + const cards = page.locator('chat-debug-checkpoint-card'); + await expect(cards.first()).toBeVisible({ timeout: 30_000 }); + await expect(cards).toHaveCount(EXPECTED_CHECKPOINT_ROWS.length); + expect((await cards.allTextContents()).map((t) => t.trim())).toEqual( + EXPECTED_CHECKPOINT_ROWS, + ); + await expect(page.getByText('No checkpoints yet.')).toHaveCount(0); +}); + +test('c-debug: selecting a checkpoint diffs that step of the run', async ({ page }) => { + await submitAndWaitForResponse(page, PROMPT); + await openDock(page); + + const cards = page.locator('chat-debug-checkpoint-card'); + await expect(cards.first()).toBeVisible({ timeout: 30_000 }); + await cards.first().click(); + + // The newest checkpoint has no predecessor in the list, so its diff is the + // whole of that checkpoint's values added at once. `messages` is the only + // key on this graph's MessagesState, and it is read from the checkpoint the + // server persisted — so a diff naming it proves the panel is rendering real + // run state rather than a placeholder. + const diff = page.locator('chat-debug-state-diff'); + await expect(diff).toBeVisible(); + await expect(diff).toContainText('+ messages'); +}); + +test('c-debug: the State tab swaps in the live state inspector', async ({ page }) => { + await submitAndWaitForResponse(page, PROMPT); + await openDock(page); + await expect(page.locator('chat-debug-checkpoint-card').first()).toBeVisible({ + timeout: 30_000, + }); + + await page.getByRole('tab', { name: 'State' }).click(); + + const stateTab = page.locator('chat-debug-state-tab'); + await expect(stateTab).toBeVisible(); + await expect(stateTab).toContainText('Current state'); + // The tab owns the panel body — the timeline is torn down, not stacked. + await expect(page.locator('chat-debug-checkpoint-card')).toHaveCount(0); + // `agent.state()` is the LangGraph values bag with `messages` projected out + // into the transcript, so on this MessagesState graph the inspector renders + // an empty object today. Assert the shape the JsonPipe produces rather than + // that exact literal: the claim is that the inspector is mounted and bound + // to the agent, and a graph that carries state beyond its messages should + // widen this tab's coverage, not fail it. + await expect(stateTab.locator('chat-debug-state-inspector pre')).toHaveText( + /^\{[\s\S]*\}$/, + ); +}); diff --git a/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json b/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json new file mode 100644 index 000000000..be0972183 --- /dev/null +++ b/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "systemMessage": "Aviation Assistant", + "userMessage": "What is a jet bridge?" + }, + "response": { + "content": "A jet bridge is the enclosed, movable walkway that connects an airport gate to an aircraft door, so passengers board without crossing the ramp." + } + }, + { + "match": { + "systemMessage": "brief one-sentence summary", + "userMessage": "What is a jet bridge?" + }, + "response": { + "content": "The traveler asked what a jet bridge is and received a short definition of the boarding walkway." + } + }, + { + "match": { + "systemMessage": "In 3-5 words", + "userMessage": "What is a jet bridge?" + }, + "response": { + "content": "Jet bridge basics" + } + } + ] +} diff --git a/cockpit/chat/debug/angular/e2e/global-setup-impl.ts b/cockpit/chat/debug/angular/e2e/global-setup-impl.ts new file mode 100644 index 000000000..91416c517 --- /dev/null +++ b/cockpit/chat/debug/angular/e2e/global-setup-impl.ts @@ -0,0 +1,15 @@ +import { resolve } from 'node:path'; +import { portsFor } from '../../../../../cockpit/ports.mjs'; +import { createGlobalSetup } from '@threadplane-internal/e2e-harness'; + +const ports = portsFor('cockpit-chat-debug-angular'); + +export default createGlobalSetup({ + // Each chat cap runs its OWN standalone backend (cockpit/chat//python) + // on ` + 1000`. The proxy.conf.mjs target matches. + langgraphCwd: 'cockpit/chat/debug/python', + langgraphPort: ports.langgraph, + angularProject: 'cockpit-chat-debug-angular', + angularPort: ports.angular, + fixturesDir: resolve(__dirname, 'fixtures'), +}); diff --git a/cockpit/chat/debug/angular/e2e/playwright.config.ts b/cockpit/chat/debug/angular/e2e/playwright.config.ts new file mode 100644 index 000000000..1c73c7420 --- /dev/null +++ b/cockpit/chat/debug/angular/e2e/playwright.config.ts @@ -0,0 +1,20 @@ +import { defineConfig, devices } from '@playwright/test'; +import { portsFor } from '../../../../../cockpit/ports.mjs'; + +const { angular: angularPort } = portsFor('cockpit-chat-debug-angular'); + +export default defineConfig({ + testDir: '.', + testMatch: '**/*.spec.ts', + fullyParallel: false, + workers: 1, + retries: process.env.CI ? 2 : 0, + reporter: process.env.CI ? [['list'], ['html', { open: 'never' }]] : 'list', + use: { + baseURL: `http://localhost:${angularPort}`, + trace: 'retain-on-failure', + }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), +}); diff --git a/cockpit/chat/debug/angular/e2e/tsconfig.json b/cockpit/chat/debug/angular/e2e/tsconfig.json new file mode 100644 index 000000000..0fc9befb1 --- /dev/null +++ b/cockpit/chat/debug/angular/e2e/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "Bundler", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "noEmit": true, + "types": [ + "node" + ], + "baseUrl": "../../../../..", + "paths": { + "@threadplane-internal/e2e-harness": [ + "libs/e2e-harness/src/index.ts" + ], + "@threadplane-internal/e2e-harness/global-teardown": [ + "libs/e2e-harness/src/global-teardown.ts" + ] + }, + "allowJs": true + }, + "include": [ + "**/*.ts" + ], + "exclude": [ + "node_modules", + "test-results", + "playwright-report" + ] +} diff --git a/cockpit/chat/debug/angular/project.json b/cockpit/chat/debug/angular/project.json index 160956c53..956991e37 100644 --- a/cockpit/chat/debug/angular/project.json +++ b/cockpit/chat/debug/angular/project.json @@ -87,6 +87,12 @@ "cwd": "cockpit/chat/debug/angular", "command": "npx tsx -e \"import { chatDebugAngularModule } from './src/index.ts'; const module = chatDebugAngularModule; if (module.id !== 'chat-debug-angular' || module.title !== 'Chat Debug (Angular)') { throw new Error('Unexpected module shape for ' + module.id); }\"" } + }, + "e2e": { + "executor": "@nx/playwright:playwright", + "options": { + "config": "cockpit/chat/debug/angular/e2e/playwright.config.ts" + } } }, "tags": [ From 1ad3e99be0097c041b1967f22beb95186b41f05e Mon Sep 17 00:00:00 2001 From: Brian Love Date: Tue, 8 Sep 2026 07:59:19 -0700 Subject: [PATCH 2/2] chore(cockpit): retire the c-debug "no aimock e2e" cross-references Two files pointed at the c-debug README as the marker for a cap that deliberately ships without an aimock suite. That is no longer true. `scripts/rerecord-all-aimock.sh` is the load-bearing one: it discovers caps by walking `*/e2e/fixtures/*.json`, so c-debug is picked up automatically now that it has a fixture, and an unlisted cap defaults to the prompt "Hello". Re-recording would have quietly replaced the fixture with one whose user message no cap spec sends. Register c-debug's actual prompt. Co-Authored-By: Claude Opus 5 --- cockpit/render/README.md | 2 +- scripts/rerecord-all-aimock.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cockpit/render/README.md b/cockpit/render/README.md index 86b8c6be6..ca44c0ade 100644 --- a/cockpit/render/README.md +++ b/cockpit/render/README.md @@ -17,4 +17,4 @@ Existing coverage: If a render-specific e2e harness is desired in the future (visual diffs, interactive scrubbing), it would be a separate cycle. The aimock pattern from cockpit-chat / cockpit-langgraph caps does not fit. -This file is the deliberate "no e2e" marker matching the c-debug README (`cockpit/chat/debug/angular/e2e/README.md`). +This file is the deliberate "no e2e" marker for the render caps. It is the only one left: `c-debug` carried the same marker until it gained a real aimock suite at `cockpit/chat/debug/angular/e2e/`. diff --git a/scripts/rerecord-all-aimock.sh b/scripts/rerecord-all-aimock.sh index b9d470ed0..753e8d779 100755 --- a/scripts/rerecord-all-aimock.sh +++ b/scripts/rerecord-all-aimock.sh @@ -42,11 +42,12 @@ declare -A CAP_PROMPTS=( ["c-subagents"]="Plan a trip from LAX to JFK" ["c-generative-ui"]="Show me a dashboard of airline operations.|Filter to only the cancelled flights." ["c-a2ui"]="I want to fly LAX to JFK|I want to fly SFO to SEA" + ["c-debug"]="What is a jet bridge?" ["streaming"]="Tell me one quick fact about Angular signals in two sentences." ) # Discover aimock-eligible caps by walking fixture files. Excludes -# documented-N/A caps (render, ag-ui, c-debug) which have no fixtures. +# documented-N/A caps (render, ag-ui) which have no fixtures. CAPS=() while IFS= read -r f; do cap_id=$(basename "$f" .json)