Skip to content

Commit 03c971a

Browse files
waleedlatif1waleedlatif1
committed
fix(browser): preserve observations and support native form controls
1 parent 1396496 commit 03c971a

10 files changed

Lines changed: 773 additions & 55 deletions

File tree

‎apps/desktop/e2e/browser-tools.spec.ts‎

Lines changed: 92 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,15 @@ const SCOPE = 'browser-tools-e2e'
1818
const FORM = `<!doctype html><html><head><title>Form fixture</title></head><body>
1919
<label>Name <input id="name" autocomplete="off"></label>
2020
<label>Plan <select id="plan" aria-label="Plan"><option value="basic">Basic</option><option value="pro">Pro</option></select></label>
21+
<label>Regions <select id="regions" aria-label="Regions" multiple><option value="a">A</option><option value="b">B</option><option value="c" disabled>C</option></select></label>
2122
<label>Updates <input id="updates" type="checkbox"></label>
23+
<label>Date <input id="date" type="date" oninput="this.dataset.events = Number(this.dataset.events || 0) + 1"></label>
24+
<label>Time <input id="time" type="time"></label>
25+
<label>Appointment <input id="appointment" type="datetime-local"></label>
26+
<label>Month <input id="month" type="month"></label>
27+
<label>Week <input id="week" type="week"></label>
28+
<label>Color <input id="color" type="color"></label>
29+
<label>Range <input id="range" type="range"></label>
2230
<label>Password <input id="password" type="password"></label>
2331
<label>Route <input id="route" oninput="history.pushState({}, '', '/form?changed=1')"></label>
2432
<a href="/redirect">Other website</a>
@@ -144,7 +152,9 @@ test.describe('browser tools', () => {
144152
const result = response.result as { snapshot: { outline: string } }
145153
expect(result.snapshot.outline).toContain('Name')
146154
return (name: string) => {
147-
const line = result.snapshot.outline.split('\n').find((line) => line.includes(`"${name}"`))
155+
const line = result.snapshot.outline
156+
.split('\n')
157+
.find((line) => line.includes(`"${name}"`) && /\[ref=\d+\]/.test(line))
148158
const match = line?.match(/\[ref=(\d+)\]/)
149159
if (!match) throw new Error(`No reference for ${name}: ${result.snapshot.outline}`)
150160
return Number(match[1])
@@ -168,6 +178,87 @@ test.describe('browser tools', () => {
168178
}, origin)
169179
}
170180

181+
test('sets and clears multiple selections without partial writes for invalid options', async () => {
182+
const ref = await openForm()
183+
const selected = await execute('browser_select_option', {
184+
elementId: ref('Regions'),
185+
values: ['A', 'B'],
186+
})
187+
expect(selected.ok, selected.error).toBe(true)
188+
expect(selected.result).toMatchObject({
189+
values: ['a', 'b'],
190+
effectObserved: true,
191+
readback: { values: ['a', 'b'] },
192+
})
193+
const invalid = await execute('browser_select_option', {
194+
elementId: ref('Regions'),
195+
values: ['B', 'C'],
196+
})
197+
expect(invalid.ok).toBe(false)
198+
const values = await app.evaluate(async ({ webContents }, origin) => {
199+
const contents = webContents
200+
.getAllWebContents()
201+
.find((wc) => wc.getURL() === `${origin}/form`)
202+
if (!contents) throw new Error('Missing form fixture')
203+
return contents.executeJavaScript(
204+
'Array.from(document.getElementById("regions").selectedOptions, option => option.value)'
205+
)
206+
}, origin)
207+
expect(values).toEqual(['a', 'b'])
208+
const cleared = await execute('browser_select_option', {
209+
elementId: ref('Regions'),
210+
values: [],
211+
})
212+
expect(cleared.result).toMatchObject({
213+
values: [],
214+
effectObserved: true,
215+
readback: { values: [] },
216+
})
217+
})
218+
219+
test('fills structured native fields and leaves invalid dates unchanged', async () => {
220+
const ref = await openForm()
221+
for (const [name, text] of [
222+
['Date', '2026-09-15'],
223+
['Time', '15:48'],
224+
['Appointment', '2026-09-15T15:48:00'],
225+
['Month', '2026-09'],
226+
['Week', '2026-W38'],
227+
['Color', '#AABBCC'],
228+
['Range', '75'],
229+
]) {
230+
const response = await execute('browser_type', { elementId: ref(name), text })
231+
expect(response.ok, response.error).toBe(true)
232+
expect(response.result).toMatchObject({
233+
trusted: false,
234+
dispatched: true,
235+
effectObserved: true,
236+
})
237+
}
238+
const invalid = await execute('browser_type', { elementId: ref('Date'), text: '2026-02-30' })
239+
expect(invalid.ok).toBe(false)
240+
expect(invalid.error).toContain('Invalid value')
241+
const state = await app.evaluate(async ({ webContents }, origin) => {
242+
const contents = webContents
243+
.getAllWebContents()
244+
.find((wc) => wc.getURL() === `${origin}/form`)
245+
if (!contents) throw new Error('Missing form fixture')
246+
return contents.executeJavaScript(
247+
'({date:document.getElementById("date").value,time:document.getElementById("time").value,appointment:document.getElementById("appointment").value,month:document.getElementById("month").value,week:document.getElementById("week").value,color:document.getElementById("color").value,range:document.getElementById("range").value,events:document.getElementById("date").dataset.events})'
248+
)
249+
}, origin)
250+
expect(state).toEqual({
251+
date: '2026-09-15',
252+
time: '15:48',
253+
appointment: '2026-09-15T15:48',
254+
month: '2026-09',
255+
week: '2026-W38',
256+
color: '#aabbcc',
257+
range: '75',
258+
events: '1',
259+
})
260+
})
261+
171262
for (const mode of ['menu', 'sticky']) {
172263
test(`clicks a ${mode} target without losing its identity`, async () => {
173264
const opened = await execute('browser_open_url', { url: `${origin}/click?mode=${mode}` })

‎apps/desktop/src/main/browser-agent/cdp.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ describe('browser-agent screenshot capture', () => {
524524

525525
await captureScreenshot(contents)
526526

527-
expect(contents.capturePage).toHaveBeenCalledWith(undefined, { stayHidden: false })
527+
expect(contents.capturePage).toHaveBeenCalledWith(undefined, { stayHidden: true })
528528
expect(contents.debugger.sendCommand).not.toHaveBeenCalledWith(
529529
'Page.captureScreenshot',
530530
expect.anything()
@@ -536,7 +536,7 @@ describe('browser-agent screenshot capture', () => {
536536

537537
const shot = await captureScreenshot(contents, { x: 100, y: 50, width: 200, height: 100 })
538538

539-
expect(contents.capturePage).toHaveBeenCalledWith(undefined, { stayHidden: false })
539+
expect(contents.capturePage).toHaveBeenCalledWith(undefined, { stayHidden: true })
540540
expect(contents.debugger.sendCommand).not.toHaveBeenCalledWith(
541541
'Page.captureScreenshot',
542542
expect.anything()

‎apps/desktop/src/main/browser-agent/cdp.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -479,7 +479,7 @@ async function captureViewportImage(
479479
})
480480
const capture = (async () => {
481481
try {
482-
return await contents.capturePage(undefined, { stayHidden: false })
482+
return await contents.capturePage(undefined, { stayHidden: true })
483483
} finally {
484484
pendingScreenshotCaptures.delete(contents)
485485
}

‎apps/desktop/src/main/browser-agent/driver.test.ts‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2145,6 +2145,50 @@ describe('credential protection', () => {
21452145
return { contents, values, writes, dialogs, selectionReads: () => selectionReads }
21462146
}
21472147

2148+
it.each([
2149+
{ values: ['a', 'b'], labels: ['A', 'B'], expected: true },
2150+
{ values: ['a'], labels: ['A'], expected: false },
2151+
{ values: ['a', 'b'], labels: ['A', 'Other'], expected: false },
2152+
])(
2153+
'verifies the entire multiple selection %j',
2154+
async ({ values: readbackValues, labels, expected }) => {
2155+
const contents = await openPage()
2156+
respondWith(contents, {
2157+
selectOptionInElement: {
2158+
selected: 'A',
2159+
value: 'a',
2160+
values: ['a', 'b'],
2161+
labels: ['A', 'B'],
2162+
},
2163+
readSelectElementState: { selected: 'A', value: 'a', values: readbackValues, labels },
2164+
})
2165+
const result = await driver.executeTool('chat-test', 'browser_select_option', {
2166+
elementId: 0,
2167+
values: ['a', 'b'],
2168+
})
2169+
expect(result, JSON.stringify(result)).toMatchObject({
2170+
ok: true,
2171+
result: { effectObserved: expected, readback: { values: readbackValues } },
2172+
})
2173+
}
2174+
)
2175+
2176+
it.each([
2177+
{ value: 'a', values: ['b'] },
2178+
{ values: [1] },
2179+
{ values: Array.from({ length: 101 }, () => 'a') },
2180+
{},
2181+
])('rejects invalid selection arguments before dispatch', async (params) => {
2182+
const contents = await openPage()
2183+
vi.mocked(contents.executeJavaScript).mockClear()
2184+
const result = await driver.executeTool('chat-test', 'browser_select_option', {
2185+
elementId: 0,
2186+
...params,
2187+
})
2188+
expect(result.ok).toBe(false)
2189+
expect(contents.executeJavaScript).not.toHaveBeenCalled()
2190+
})
2191+
21482192
const formFields = [
21492193
{ elementId: 1, kind: 'select', value: 'first' },
21502194
{ elementId: 2, kind: 'select', value: 'second' },
@@ -2475,6 +2519,97 @@ describe('credential protection', () => {
24752519
expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(1)
24762520
})
24772521

2522+
it('sets structured input values without dispatching text or select-all keystrokes', async () => {
2523+
const contents = await openPage()
2524+
respondWith(contents, {
2525+
focusElementForTyping: { focused: true, kind: 'input', valueInput: true, x: 24, y: 48 },
2526+
setFocusedInputValue: { dispatched: true },
2527+
readActiveElementState: { activeElement: 'input', valueLength: 10 },
2528+
readPageActionState: {},
2529+
})
2530+
const result = await driver.executeTool('chat-test', 'browser_type', {
2531+
elementId: 0,
2532+
text: '2026-09-15',
2533+
})
2534+
expect(result).toMatchObject({ ok: true, result: { dispatched: true, trusted: false } })
2535+
expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0)
2536+
expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(0)
2537+
})
2538+
2539+
it('does not retry a rejected structured value through synthetic typing', async () => {
2540+
const contents = await openPage()
2541+
respondWith(contents, {
2542+
focusElementForTyping: { focused: true, kind: 'input', valueInput: true, x: 24, y: 48 },
2543+
setFocusedInputValue: { error: 'Invalid value; the field was not changed.' },
2544+
readActiveElementState: {},
2545+
readPageActionState: {},
2546+
})
2547+
const result = await driver.executeTool('chat-test', 'browser_type', {
2548+
elementId: 0,
2549+
text: 'invalid-date',
2550+
})
2551+
expect(result).toMatchObject({ ok: false, error: expect.stringContaining('Invalid value') })
2552+
expect(
2553+
vi
2554+
.mocked(contents.executeJavaScript)
2555+
.mock.calls.filter(([expression]) => isPageCall(String(expression), 'typeIntoElement'))
2556+
).toHaveLength(0)
2557+
expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0)
2558+
})
2559+
2560+
it('reports an interrupted structured write as uncertain without replaying it', async () => {
2561+
const contents = await openPage()
2562+
let writes = 0
2563+
vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
2564+
if (isPageCall(expression, 'focusElementForTyping'))
2565+
return Promise.resolve({ focused: true, valueInput: true, x: 24, y: 48 })
2566+
if (isPageCall(expression, 'setFocusedInputValue')) {
2567+
writes++
2568+
return Promise.reject(new Error('Execution context was destroyed'))
2569+
}
2570+
return Promise.resolve({})
2571+
})
2572+
const result = await driver.executeTool('chat-test', 'browser_type', {
2573+
elementId: 0,
2574+
text: '2026-09-15',
2575+
})
2576+
expect(result).toMatchObject({
2577+
ok: false,
2578+
error: expect.stringContaining('may have reached the field and was not retried'),
2579+
})
2580+
expect(writes).toBe(1)
2581+
expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0)
2582+
expect(
2583+
vi
2584+
.mocked(contents.executeJavaScript)
2585+
.mock.calls.some(([expression]) => isPageCall(String(expression), 'typeIntoElement'))
2586+
).toBe(false)
2587+
})
2588+
2589+
it('refuses a field whose input mode changes before dispatch', async () => {
2590+
const contents = await openPage()
2591+
let reads = 0
2592+
vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
2593+
if (isPageCall(expression, 'focusElementForTyping'))
2594+
return Promise.resolve({ focused: true, valueInput: ++reads === 1, x: 24, y: 48 })
2595+
return Promise.resolve({})
2596+
})
2597+
const result = await driver.executeTool('chat-test', 'browser_type', {
2598+
elementId: 0,
2599+
text: '2026-09-15',
2600+
})
2601+
expect(result).toMatchObject({
2602+
ok: false,
2603+
error: expect.stringContaining('field type changed'),
2604+
})
2605+
expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0)
2606+
expect(
2607+
vi
2608+
.mocked(contents.executeJavaScript)
2609+
.mock.calls.some(([expression]) => isPageCall(String(expression), 'setFocusedInputValue'))
2610+
).toBe(false)
2611+
})
2612+
24782613
it('accepts empty text and sends it through native insertion to clear a field', async () => {
24792614
const contents = await openPage()
24802615
respondWith(contents, {

0 commit comments

Comments
 (0)