Skip to content

Commit fb3faf7

Browse files
pratikbuildspratikbuilds
committed
feat(media): add generated images, sources and MediaViewer
GeneratedImages holds each tile at its final size under the dither grain while it generates, with the elapsed time beside the label. An image made while watched is revealed by the grain; one already there loads plainly; a failed one shows a message and a retry. The grain reads its colours from existing tokens and draws one still frame under reduced motion. Sources lists citations, and MediaViewer shows images on the native dialog with arrow keys between items and focus returned to the opener. Media addresses from agent data are checked before use.
1 parent dfa43c8 commit fb3faf7

16 files changed

Lines changed: 2463 additions & 0 deletions

‎e2e/fixtures.tsx‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,12 @@ import {
7272
type AttachmentItem,
7373
type AttachmentRejection,
7474
useAttachments,
75+
GeneratedImages,
76+
generatedImageViewerItems,
77+
MediaViewer,
78+
useMediaViewer,
79+
type ConversationImage,
80+
type PartImage,
7581
} from "@corbits/react-ui";
7682
import {
7783
CommandPalette,
@@ -1198,6 +1204,79 @@ function AttachmentsScenario() {
11981204
);
11991205
}
12001206

1207+
/** A flat swatch as an inline SVG, so nothing is fetched. */
1208+
function swatch(index: number): ConversationImage {
1209+
const shade = 40 + index * 20;
1210+
const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="300" height="200"><rect width="300" height="200" fill="hsl(${shade} 70% 60%)"/></svg>`;
1211+
return {
1212+
id: `swatch-${index}`,
1213+
mediaType: "image/svg+xml",
1214+
width: 300,
1215+
height: 200,
1216+
alt: `Swatch ${index}`,
1217+
url: `data:image/svg+xml,${encodeURIComponent(svg)}`,
1218+
};
1219+
}
1220+
1221+
const SWATCHES: PartImage = {
1222+
kind: "image",
1223+
id: "swatches",
1224+
prompt: "Three swatches",
1225+
state: "done",
1226+
images: [swatch(1), swatch(2), swatch(3)],
1227+
};
1228+
1229+
/** Starts generating, so landing the images is watched and reveals them. */
1230+
function MediaViewerScenario() {
1231+
const [landed, setLanded] = useState(false);
1232+
const part: PartImage = landed
1233+
? SWATCHES
1234+
: {
1235+
...SWATCHES,
1236+
state: "generating",
1237+
images: [],
1238+
expected: { count: 3, width: 300, height: 200 },
1239+
};
1240+
const viewer = useMediaViewer({ items: generatedImageViewerItems(part) });
1241+
return (
1242+
<div style={{ width: 600 }}>
1243+
<Button onClick={() => setLanded(true)}>Land images</Button>
1244+
<GeneratedImages part={part} onOpen={viewer.open} />
1245+
<MediaViewer viewer={viewer} />
1246+
</div>
1247+
);
1248+
}
1249+
1250+
const REMOTE: PartImage = {
1251+
kind: "image",
1252+
id: "remote",
1253+
prompt: "Remote swatches",
1254+
state: "done",
1255+
images: [1, 2].map((index) => ({
1256+
id: `remote-${index}`,
1257+
mediaType: "image/svg+xml",
1258+
width: 300,
1259+
height: 200,
1260+
alt: `Remote ${index}`,
1261+
url: `https://images.example.test/remote-${index}.svg`,
1262+
})),
1263+
};
1264+
1265+
/** Images from another origin inside a scrolling log, on a page that scrolls. */
1266+
function MediaViewerChatScenario() {
1267+
const viewer = useMediaViewer({ items: generatedImageViewerItems(REMOTE) });
1268+
return (
1269+
<div style={{ width: 600, paddingBottom: 3000 }}>
1270+
<div data-testid="log" style={{ height: 320, overflow: "auto" }}>
1271+
<div style={{ height: 600 }} />
1272+
<GeneratedImages part={REMOTE} onOpen={viewer.open} />
1273+
<div style={{ height: 600 }} />
1274+
</div>
1275+
<MediaViewer viewer={viewer} />
1276+
</div>
1277+
);
1278+
}
1279+
12011280
export const scenarios: Record<string, () => ReactNode> = {
12021281
smoke: () => <Button>Ready</Button>,
12031282
"approval-card": () => <Approval state="idle" />,
@@ -1262,4 +1341,6 @@ export const scenarios: Record<string, () => ReactNode> = {
12621341
</>
12631342
),
12641343
attachments: () => <AttachmentsScenario />,
1344+
"media-viewer": () => <MediaViewerScenario />,
1345+
"media-viewer-chat": () => <MediaViewerChatScenario />,
12651346
};

‎e2e/media-viewer.spec.ts‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import { expect, test, type Page } from "@playwright/test";
2+
3+
const grain = (page: Page) =>
4+
page.locator("[data-slot='generated-image-grain']");
5+
6+
async function land(page: Page) {
7+
await page.goto("/media-viewer");
8+
await expect(
9+
page.locator("[data-slot='generated-image-placeholder']"),
10+
).toHaveCount(3);
11+
await expect(page.getByRole("status")).toHaveText("Generating 3 images");
12+
await page.getByRole("button", { name: "Land images" }).click();
13+
}
14+
15+
test("images made while watched are revealed by the grain", async ({
16+
page,
17+
}) => {
18+
await land(page);
19+
await expect(grain(page)).toHaveCount(3);
20+
await expect(page.getByRole("status")).toHaveText("Made 3 images");
21+
// The reveal is one 700ms pass per tile, cascading 90ms apart.
22+
await expect(grain(page)).toHaveCount(0, { timeout: 3_000 });
23+
const tiles = page.locator("[data-slot='generated-image']");
24+
await expect(tiles).toHaveCount(3);
25+
for (const tile of await tiles.all()) {
26+
await expect(tile).toHaveAttribute("data-state", "loaded");
27+
}
28+
});
29+
30+
test("with reduced motion, landed images skip the grain", async ({ page }) => {
31+
await page.emulateMedia({ reducedMotion: "reduce" });
32+
await land(page);
33+
await expect(
34+
page.locator("[data-slot='generated-image'][data-state='loaded']"),
35+
).toHaveCount(3);
36+
await expect(grain(page)).toHaveCount(0);
37+
});
38+
39+
test("arrow keys move between items, Escape closes, focus returns to the tile", async ({
40+
page,
41+
}) => {
42+
await land(page);
43+
await expect(grain(page)).toHaveCount(0, { timeout: 3_000 });
44+
const tile = page.getByRole("button", {
45+
name: "Open Swatch 2",
46+
exact: true,
47+
});
48+
await tile.click();
49+
50+
const dialog = page.getByRole("dialog");
51+
await expect(dialog).toHaveAccessibleName("three-swatches-2.svg");
52+
await expect(page.getByRole("button", { name: "Close" })).toBeFocused();
53+
await expect(dialog.getByRole("img")).toHaveAttribute("alt", "Swatch 2");
54+
55+
await page.keyboard.press("ArrowRight");
56+
await expect(dialog).toHaveAccessibleName("three-swatches-3.svg");
57+
await expect(dialog.getByRole("status")).toHaveText(
58+
"three-swatches-3.svg, 3 of 3",
59+
);
60+
// Stepping stops at the ends rather than wrapping.
61+
await page.keyboard.press("ArrowRight");
62+
await expect(dialog).toHaveAccessibleName("three-swatches-3.svg");
63+
await expect(dialog.getByRole("button", { name: "Next" })).toHaveAttribute(
64+
"aria-disabled",
65+
"true",
66+
);
67+
68+
await page.keyboard.press("ArrowLeft");
69+
await page.keyboard.press("ArrowLeft");
70+
await expect(dialog).toHaveAccessibleName("three-swatches-1.svg");
71+
await page.keyboard.press("ArrowLeft");
72+
await expect(dialog).toHaveAccessibleName("three-swatches-1.svg");
73+
74+
await page.keyboard.press("Escape");
75+
await expect(dialog).toHaveCount(0);
76+
await expect(tile).toBeFocused();
77+
});
78+
79+
test("the viewer opens from the keyboard and closes from its button or the backdrop", async ({
80+
page,
81+
}) => {
82+
await land(page);
83+
await expect(grain(page)).toHaveCount(0, { timeout: 3_000 });
84+
const tile = page.getByRole("button", {
85+
name: "Open Swatch 3",
86+
exact: true,
87+
});
88+
await tile.focus();
89+
await page.keyboard.press("Enter");
90+
const dialog = page.getByRole("dialog");
91+
await expect(dialog).toHaveAccessibleName("three-swatches-3.svg");
92+
await dialog.getByRole("button", { name: "Previous" }).click();
93+
await expect(dialog).toHaveAccessibleName("three-swatches-2.svg");
94+
await dialog.getByRole("button", { name: "Close" }).click();
95+
await expect(page.getByRole("dialog")).toHaveCount(0);
96+
await expect(tile).toBeFocused();
97+
98+
// The action button opens it too, and a click outside the media closes it.
99+
const action = page.getByRole("button", {
100+
name: "Open Swatch 1 full size",
101+
});
102+
await action.focus();
103+
await page.keyboard.press("Enter");
104+
await expect(page.getByRole("dialog")).toHaveAccessibleName(
105+
"three-swatches-1.svg",
106+
);
107+
await page.mouse.click(4, 4);
108+
await expect(page.getByRole("dialog")).toHaveCount(0);
109+
await expect(action).toBeFocused();
110+
});
111+
112+
const REMOTE_SVG =
113+
'<svg xmlns="http://www.w3.org/2000/svg" width="300" height="200"><rect width="300" height="200" fill="gray"/></svg>';
114+
115+
async function openRemote(page: Page) {
116+
await page
117+
.context()
118+
.route("https://images.example.test/**", (route) =>
119+
route.fulfill({ contentType: "image/svg+xml", body: REMOTE_SVG }),
120+
);
121+
await page.goto("/media-viewer-chat");
122+
await page
123+
.getByRole("button", { name: "Open Remote 2", exact: true })
124+
.click();
125+
const dialog = page.getByRole("dialog");
126+
await expect(dialog).toHaveAccessibleName("remote-swatches-2.svg");
127+
return dialog;
128+
}
129+
130+
const scrolls = (page: Page) =>
131+
page.evaluate(() => ({
132+
page: document.scrollingElement?.scrollTop ?? 0,
133+
log: document.querySelector("[data-testid='log']")?.scrollTop ?? 0,
134+
}));
135+
136+
test("with the viewer open, the wheel scrolls neither the page nor the log behind it", async ({
137+
page,
138+
}) => {
139+
await openRemote(page);
140+
const before = await scrolls(page);
141+
expect(before.log).toBeGreaterThan(0);
142+
// Over the backdrop, the picture and the bar, in both directions.
143+
for (const [x, y] of [
144+
[640, 360],
145+
[20, 20],
146+
[640, 640],
147+
] as const) {
148+
await page.mouse.move(x, y);
149+
await page.mouse.wheel(0, 800);
150+
await page.mouse.wheel(0, -400);
151+
}
152+
await page.waitForTimeout(300);
153+
expect(await scrolls(page)).toEqual(before);
154+
155+
// Closing hands scrolling back to the page.
156+
await page.keyboard.press("Escape");
157+
await expect(page.getByRole("dialog")).toHaveCount(0);
158+
expect(
159+
await page.evaluate(() => document.documentElement.style.overflow),
160+
).toBe("");
161+
await page.mouse.move(640, 700);
162+
await page.mouse.wheel(0, 400);
163+
await expect
164+
.poll(async () => (await scrolls(page)).page)
165+
.toBeGreaterThan(before.page);
166+
});
167+
168+
test("Download of an image from another origin leaves the chat in place", async ({
169+
page,
170+
}) => {
171+
const dialog = await openRemote(page);
172+
const popup = page.waitForEvent("popup");
173+
await dialog
174+
.getByRole("link", { name: "Download remote-swatches-2.svg" })
175+
.click();
176+
await popup;
177+
expect(new URL(page.url()).pathname).toBe("/media-viewer-chat");
178+
await expect(dialog).toBeVisible();
179+
});

‎src/hooks/use-media-viewer.test.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { act } from "react";
2+
import { describe, expect, test } from "bun:test";
3+
4+
import { renderHook } from "../test/render-hook.js";
5+
import {
6+
useMediaViewer,
7+
type UseMediaViewerOptions,
8+
} from "./use-media-viewer.js";
9+
10+
const ITEMS = ["a", "b", "c"] as const;
11+
12+
function viewer(options: Partial<UseMediaViewerOptions<string>> = {}) {
13+
return renderHook(() => useMediaViewer<string>({ items: ITEMS, ...options }));
14+
}
15+
16+
describe("useMediaViewer", () => {
17+
test("starts closed and opens an item", () => {
18+
const hook = viewer();
19+
expect(hook.result.current.index).toBe(null);
20+
expect(hook.result.current.item).toBe(null);
21+
act(() => hook.result.current.open(1));
22+
expect(hook.result.current.index).toBe(1);
23+
expect(hook.result.current.item).toBe("b");
24+
hook.unmount();
25+
});
26+
27+
test("next and previous stop at the ends", () => {
28+
const hook = viewer({ defaultIndex: 1 });
29+
act(() => hook.result.current.next());
30+
expect(hook.result.current.index).toBe(2);
31+
expect(hook.result.current.hasNext).toBe(false);
32+
act(() => hook.result.current.next());
33+
expect(hook.result.current.index).toBe(2);
34+
35+
act(() => hook.result.current.previous());
36+
act(() => hook.result.current.previous());
37+
expect(hook.result.current.index).toBe(0);
38+
expect(hook.result.current.hasPrevious).toBe(false);
39+
act(() => hook.result.current.previous());
40+
expect(hook.result.current.index).toBe(0);
41+
expect(hook.result.current.hasNext).toBe(true);
42+
hook.unmount();
43+
});
44+
45+
test("closing clears the open item", () => {
46+
const hook = viewer({ defaultIndex: 2 });
47+
act(() => hook.result.current.close());
48+
expect(hook.result.current.index).toBe(null);
49+
expect(hook.result.current.item).toBe(null);
50+
expect(hook.result.current.hasNext).toBe(false);
51+
expect(hook.result.current.hasPrevious).toBe(false);
52+
// Stepping does nothing while closed.
53+
act(() => hook.result.current.next());
54+
expect(hook.result.current.index).toBe(null);
55+
hook.unmount();
56+
});
57+
58+
test("an index outside the items reads as closed, and open ignores one", () => {
59+
const hook = viewer({ defaultIndex: 5 });
60+
expect(hook.result.current.index).toBe(null);
61+
act(() => hook.result.current.open(3));
62+
expect(hook.result.current.index).toBe(null);
63+
act(() => hook.result.current.open(-1));
64+
expect(hook.result.current.index).toBe(null);
65+
hook.unmount();
66+
});
67+
68+
test("controlled, it reports changes and follows the value it is given", () => {
69+
const changes: (number | null)[] = [];
70+
const hook = viewer({
71+
index: 0,
72+
onIndexChange: (index) => changes.push(index),
73+
});
74+
act(() => hook.result.current.next());
75+
act(() => hook.result.current.close());
76+
expect(changes).toEqual([1, null]);
77+
expect(hook.result.current.index).toBe(0);
78+
hook.unmount();
79+
});
80+
81+
test("keeps the element that opened it", () => {
82+
const hook = viewer();
83+
const button = document.createElement("button");
84+
act(() => hook.result.current.open(0, button));
85+
expect(hook.result.current.opener.current).toBe(button);
86+
act(() => hook.result.current.open(1));
87+
expect(hook.result.current.opener.current).toBe(null);
88+
hook.unmount();
89+
});
90+
});

0 commit comments

Comments
 (0)