diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-android-settings-fixture.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-android-settings-fixture.ts new file mode 100644 index 0000000000..4207066f2b --- /dev/null +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-android-settings-fixture.ts @@ -0,0 +1,85 @@ +import { formatRole } from '@agent-device/kernel/snapshot'; +import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; + +/** + * The scroll-container chain of the Android 16 Settings home screen (Pixel 7, API 36), trimmed from + * live `snapshot --json` captures for #3737. `main_content_scrollable_container` and the + * `recycler_view` nested inside it share one rect. The RecyclerView offers ACTION_SCROLL_BACKWARD + * even at the top, so it reports `hiddenContentAbove` in both states; only the outer ScrollView's + * hint tracks where the content actually is. + */ +export type SettingsHomeState = 'at-top' | 'scrolled-down'; + +const SCREEN: Rect = { x: 0, y: 0, width: 1440, height: 3120 }; +const LIST_FRAME: Rect = { x: 0, y: 480, width: 1440, height: 2556 }; +const ROW_TOPS: Record = { + 'at-top': [480, 807, 1078, 1405], + 'scrolled-down': [493, 764, 1035, 1362], +}; +const ROW_LABELS: Record = { + 'at-top': ['Google', 'Network & internet', 'Connected devices', 'Apps'], + 'scrolled-down': ['Network & internet', 'Connected devices', 'Apps', 'Notifications'], +}; +const RECYCLER_VIEW_INDEX = 23; + +function node( + index: number, + type: string, + rect: Rect, + overrides: Partial = {}, +): SnapshotNode { + return { ref: `e${index + 1}`, index, type, kind: formatRole(type), rect, ...overrides }; +} + +export function settingsHomeTree(state: SettingsHomeState): SnapshotNode[] { + const rows = ROW_TOPS[state].map((y, row) => + node( + RECYCLER_VIEW_INDEX + 1 + row, + 'android.widget.LinearLayout', + { x: 0, y, width: 1440, height: 271 }, + { parentIndex: RECYCLER_VIEW_INDEX, label: ROW_LABELS[state][row] }, + ), + ); + return [ + node(8, 'android.widget.FrameLayout', SCREEN), + node(10, 'android.widget.FrameLayout', SCREEN, { + parentIndex: 8, + identifier: 'android:id/content', + }), + node( + 11, + 'android.widget.ScrollView', + { x: 0, y: 0, width: 1440, height: 3036 }, + { + parentIndex: 10, + identifier: 'com.android.settings:id/settings_homepage_container', + }, + ), + node( + 12, + 'android.widget.TextView', + { x: 252, y: 251, width: 518, height: 93 }, + { + parentIndex: 11, + identifier: 'com.android.settings:id/search_bar_title', + label: 'Search Settings', + }, + ), + node(18, 'android.widget.ScrollView', LIST_FRAME, { + parentIndex: 11, + identifier: 'com.android.settings:id/main_content_scrollable_container', + hiddenContentBelow: true, + ...(state === 'scrolled-down' ? { hiddenContentAbove: true } : {}), + }), + node(22, 'android.widget.FrameLayout', LIST_FRAME, { + parentIndex: 18, + identifier: 'android:id/list_container', + }), + node(RECYCLER_VIEW_INDEX, 'androidx.recyclerview.widget.RecyclerView', LIST_FRAME, { + parentIndex: 22, + identifier: 'com.android.settings:id/recycler_view', + hiddenContentAbove: true, + }), + ...rows, + ]; +} diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-fixtures.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-fixtures.ts index 349fa7ca8d..756b938d78 100644 --- a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-fixtures.ts +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-fixtures.ts @@ -4,6 +4,7 @@ import { type ScrollEdgeTarget, } from '../scroll-edge-state.ts'; import { AppError } from '@agent-device/kernel/errors'; +import { matchesSnapshotScope, reindexSnapshotNodes } from '@agent-device/contracts/snapshot'; import { formatRole } from '@agent-device/kernel/snapshot'; import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; @@ -116,3 +117,22 @@ export async function captureThrows(scope: string | undefined): Promise matchesSnapshotScope(node, scope)); + if (!root) return []; + const inScope = new Set([root.index]); + for (const node of nodes) { + if (node.parentIndex !== undefined && inScope.has(node.parentIndex)) inScope.add(node.index); + } + return reindexSnapshotNodes(nodes.filter((node) => inScope.has(node.index))); +} diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-same-rect-ancestor.test.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-same-rect-ancestor.test.ts new file mode 100644 index 0000000000..ffff79a16e --- /dev/null +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-same-rect-ancestor.test.ts @@ -0,0 +1,105 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { runScrollEdgePasses } from '../scroll-edge-state.ts'; +import { capture, scrollNode, subtreeFor, windowRoot } from './scroll-edge-state-fixtures.ts'; +import { settingsHomeTree } from './scroll-edge-state-android-settings-fixture.ts'; + +// --------------------------------------------------------------------------- +// A scroller nested in a scrollable ancestor with the same rect takes its edge hints from that +// ancestor (#3737): the inner RecyclerView on Android Settings claims hidden content above even at +// the top, while the outer ScrollView framing the same pixels reports where the content is. +// --------------------------------------------------------------------------- + +test('scroll top edge: the Android Settings list at the top reports no hidden content above', async () => { + const state = await capture(settingsHomeTree('at-top'), 'top'); + assert.equal(state.canScroll, false); +}); + +test('scroll top on Android Settings already at the top costs no pass under scoped recapture', async () => { + // Scoped captures return only the scope root's subtree, as an Android scoped snapshot does. + const tree = settingsHomeTree('at-top'); + const outcome = await runScrollEdgePasses({ + edge: 'top', + captureState: async (scope) => await capture(subtreeFor(tree, scope), 'top'), + scroll: async () => { + throw new Error('scroll top dispatched a gesture at the top'); + }, + }); + assert.equal(outcome.passes, 0); +}); + +test('scroll top edge: a cyclic chain of same-rect scrollables ends the ancestor walk', async () => { + const rect = { x: 0, y: 100, width: 400, height: 600 }; + const nodes = [ + windowRoot(), + scrollNode(1, { parentIndex: 2, identifier: 'first', hiddenContentAbove: true, rect }), + scrollNode(2, { parentIndex: 1, identifier: 'second', rect }), + ]; + const state = await capture(nodes, 'top'); + assert.equal(state.scope, 'first'); + assert.equal(state.canScroll, true); +}); + +test('scroll top edge: the Android Settings list scrolled down still reports hidden content above', async () => { + const state = await capture(settingsHomeTree('scrolled-down'), 'top'); + assert.equal(state.canScroll, true); +}); + +test('scroll bottom edge: the Android Settings list at the top still reports hidden content below', async () => { + const state = await capture(settingsHomeTree('at-top'), 'bottom'); + assert.equal(state.canScroll, true); +}); + +test('scroll top on Android Settings stops at the top instead of reporting no progress', async () => { + const frames = [settingsHomeTree('scrolled-down'), settingsHomeTree('at-top')]; + let scrolls = 0; + const outcome = await runScrollEdgePasses({ + edge: 'top', + captureState: async () => await capture(frames[Math.min(scrolls, frames.length - 1)]!, 'top'), + scroll: async () => { + scrolls += 1; + }, + }); + assert.equal(outcome.passes, 1); +}); + +test('scroll top edge: a nested scroller whose rect differs from its scrollable ancestor keeps its own hint', async () => { + const nodes = [ + windowRoot(), + scrollNode(1, { identifier: 'outer', rect: { x: 0, y: 0, width: 400, height: 800 } }), + scrollNode(2, { + parentIndex: 1, + identifier: 'inner', + hiddenContentAbove: true, + rect: { x: 0, y: 100, width: 400, height: 600 }, + }), + ]; + const state = await capture(nodes, 'top'); + assert.equal(state.scope, 'inner'); + assert.equal(state.canScroll, true); +}); + +test('scroll top edge: a targeted node in a cyclic chain with no scrollable ancestor ends the walk', async () => { + const nodes = [ + windowRoot(), + scrollNode(1, { identifier: 'list', hiddenContentAbove: true }), + { ...scrollNode(2, { parentIndex: 3 }), type: 'Button', kind: 'button' }, + { ...scrollNode(3, { parentIndex: 2 }), type: 'Group', kind: 'group' }, + ]; + const state = await capture(nodes, 'top', { nodeIndex: 2 }); + assert.equal(state.scope, 'list'); + assert.equal(state.canScroll, true); +}); + +test('scroll bottom edge: a same-rect horizontal pager does not stand in for the vertical list on its page', async () => { + // A horizontal pager reports no vertical edge hints; its page's vertical list carries them. + const rect = { x: 0, y: 100, width: 400, height: 600 }; + const nodes = [ + windowRoot(), + scrollNode(1, { identifier: 'pager', rect }), + scrollNode(2, { parentIndex: 1, identifier: 'feed', hiddenContentBelow: true, rect }), + ]; + const state = await capture(nodes, 'bottom'); + assert.equal(state.scope, 'feed'); + assert.equal(state.canScroll, true); +}); diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts index 7ef7a1fd94..5fec0a0727 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts @@ -64,12 +64,23 @@ function selectScrollContainer( target: ScrollEdgeTarget, ): SnapshotNode | null { const visibility = createSnapshotVisibility(nodes); - const scrollables = nodes.filter((node) => isScrollableNodeLike(node) && isUsableRect(node.rect)); + const reportsVerticalEdge = (node: SnapshotNode): boolean => + hasHiddenContentAtEdge(node, hiddenHints.get(node.index), 'top') || + hasHiddenContentAtEdge(node, hiddenHints.get(node.index), 'bottom'); + const edgeContainer = (node: SnapshotNode): SnapshotNode => + sameRectScrollableAncestor(node, visibility.nodeByIndex, reportsVerticalEdge) ?? node; + const scrollables = [ + ...new Set( + nodes + .filter((node) => isScrollableNodeLike(node) && isUsableRect(node.rect)) + .map(edgeContainer), + ), + ]; if (scrollables.length === 0) return null; const targetAncestor = findNearestScrollableAncestor(target.nodeIndex, visibility.nodeByIndex); if (targetAncestor) { - return targetAncestor; + return edgeContainer(targetAncestor); } const targetPoint = target.point; @@ -136,12 +147,14 @@ function findNearestScrollableAncestor( byIndex: ReadonlyMap, ): SnapshotNode | null { if (nodeIndex === undefined) return null; + const visited = new Set(); let node = byIndex.get(nodeIndex); - while (node) { + while (node && !visited.has(node.index)) { + visited.add(node.index); if (isScrollableNodeLike(node) && isUsableRect(node.rect)) { return node; } - node = node.parentIndex === undefined ? undefined : byIndex.get(node.parentIndex); + node = parentOf(node, byIndex); } return null; } @@ -157,6 +170,54 @@ function hasHiddenContentAtEdge( return node.hiddenContentAbove === true || hint?.hiddenContentAbove === true; } +/** + * The outermost scrollable in an unbroken chain of same-rect, vertically scrolling ancestors, or + * undefined when the nearest scrollable ancestor frames different pixels or reports no vertical edge. + * Scroll edge selection stands that ancestor in for the nested scroller: Android's Settings + * RecyclerView offers scroll-backward even at the top, while the ScrollView framing the same pixels + * reports the real position (#3737). Choosing the ancestor also keeps the nested scroller inside the + * scoped recapture. Visited indexes stop a cyclic `parentIndex` chain in a malformed tree. + * + * An identical rect is a geometric heuristic for "these two scrollers move the same content", and + * geometry carries no scroll axis. Requiring the ancestor to report a vertical edge hint keeps a + * horizontal pager whose page holds a vertical list from standing in for that list; the snapshot + * emits vertical edge hints only for vertical scrollers. A vertical ancestor whose content fits in + * both directions reports no edge either, and leaves the nested scroller to answer for itself. + */ +function sameRectScrollableAncestor( + node: SnapshotNode, + byIndex: ReadonlyMap, + reportsVerticalEdge: (candidate: SnapshotNode) => boolean, +): SnapshotNode | undefined { + const visited = new Set([node.index]); + let owner: SnapshotNode | undefined; + let current = node; + let parent = parentOf(current, byIndex); + while (parent && !visited.has(parent.index)) { + visited.add(parent.index); + if (isScrollableNodeLike(parent) && isUsableRect(parent.rect)) { + if (!sameRect(parent.rect, current.rect) || !reportsVerticalEdge(parent)) return owner; + owner = parent; + current = parent; + } + parent = parentOf(parent, byIndex); + } + return owner; +} + +function parentOf( + node: SnapshotNode, + byIndex: ReadonlyMap, +): SnapshotNode | undefined { + return node.parentIndex === undefined ? undefined : byIndex.get(node.parentIndex); +} + +function sameRect(a: SnapshotNode['rect'], b: SnapshotNode['rect']): boolean { + return Boolean( + a && b && a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height, + ); +} + function buildScrollContainerScope( node: SnapshotNode, nodes: readonly SnapshotNode[],