Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
import { formatRole } from '@agent-device/kernel/snapshot';
import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';

/**
* The scroll-container chain of the Android 16 Settings home screen (Pixel 7, API 36), trimmed from
* live `snapshot --json` captures for #3737. `main_content_scrollable_container` and the
* `recycler_view` nested inside it share one rect. The RecyclerView offers ACTION_SCROLL_BACKWARD
* even at the top, so it reports `hiddenContentAbove` in both states; only the outer ScrollView's
* hint tracks where the content actually is.
*/
export type SettingsHomeState = 'at-top' | 'scrolled-down';

const SCREEN: Rect = { x: 0, y: 0, width: 1440, height: 3120 };
const LIST_FRAME: Rect = { x: 0, y: 480, width: 1440, height: 2556 };
const ROW_TOPS: Record<SettingsHomeState, readonly number[]> = {
'at-top': [480, 807, 1078, 1405],
'scrolled-down': [493, 764, 1035, 1362],
};
const ROW_LABELS: Record<SettingsHomeState, readonly string[]> = {
'at-top': ['Google', 'Network & internet', 'Connected devices', 'Apps'],
'scrolled-down': ['Network & internet', 'Connected devices', 'Apps', 'Notifications'],
};
const RECYCLER_VIEW_INDEX = 23;

function node(
index: number,
type: string,
rect: Rect,
overrides: Partial<SnapshotNode> = {},
): SnapshotNode {
return { ref: `e${index + 1}`, index, type, kind: formatRole(type), rect, ...overrides };
}

export function settingsHomeTree(state: SettingsHomeState): SnapshotNode[] {
const rows = ROW_TOPS[state].map((y, row) =>
node(
RECYCLER_VIEW_INDEX + 1 + row,
'android.widget.LinearLayout',
{ x: 0, y, width: 1440, height: 271 },
{ parentIndex: RECYCLER_VIEW_INDEX, label: ROW_LABELS[state][row] },
),
);
return [
node(8, 'android.widget.FrameLayout', SCREEN),
node(10, 'android.widget.FrameLayout', SCREEN, {
parentIndex: 8,
identifier: 'android:id/content',
}),
node(
11,
'android.widget.ScrollView',
{ x: 0, y: 0, width: 1440, height: 3036 },
{
parentIndex: 10,
identifier: 'com.android.settings:id/settings_homepage_container',
},
),
node(
12,
'android.widget.TextView',
{ x: 252, y: 251, width: 518, height: 93 },
{
parentIndex: 11,
identifier: 'com.android.settings:id/search_bar_title',
label: 'Search Settings',
},
),
node(18, 'android.widget.ScrollView', LIST_FRAME, {
parentIndex: 11,
identifier: 'com.android.settings:id/main_content_scrollable_container',
hiddenContentBelow: true,
...(state === 'scrolled-down' ? { hiddenContentAbove: true } : {}),
}),
node(22, 'android.widget.FrameLayout', LIST_FRAME, {
parentIndex: 18,
identifier: 'android:id/list_container',
}),
node(RECYCLER_VIEW_INDEX, 'androidx.recyclerview.widget.RecyclerView', LIST_FRAME, {
parentIndex: 22,
identifier: 'com.android.settings:id/recycler_view',
hiddenContentAbove: true,
}),
...rows,
];
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
type ScrollEdgeTarget,
} from '../scroll-edge-state.ts';
import { AppError } from '@agent-device/kernel/errors';
import { matchesSnapshotScope, reindexSnapshotNodes } from '@agent-device/contracts/snapshot';
import { formatRole } from '@agent-device/kernel/snapshot';
import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot';

Expand Down Expand Up @@ -116,3 +117,22 @@ export async function captureThrows(scope: string | undefined): Promise<AppError
throw error;
}
}

/**
* The scope root's subtree, reindexed, as an Android scoped snapshot returns it: the first node in
* document order whose label, value or identifier matches, plus its descendants. Unscoped returns
* the whole tree.
*/
export function subtreeFor(
nodes: readonly SnapshotNode[],
scope: string | undefined,
): SnapshotNode[] {
if (scope === undefined) return [...nodes];
const root = nodes.find((node) => matchesSnapshotScope(node, scope));
if (!root) return [];
const inScope = new Set([root.index]);
for (const node of nodes) {
if (node.parentIndex !== undefined && inScope.has(node.parentIndex)) inScope.add(node.index);
}
return reindexSnapshotNodes(nodes.filter((node) => inScope.has(node.index)));
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import { test } from 'vitest';
import assert from 'node:assert/strict';
import { runScrollEdgePasses } from '../scroll-edge-state.ts';
import { capture, scrollNode, subtreeFor, windowRoot } from './scroll-edge-state-fixtures.ts';
import { settingsHomeTree } from './scroll-edge-state-android-settings-fixture.ts';

// ---------------------------------------------------------------------------
// A scroller nested in a scrollable ancestor with the same rect takes its edge hints from that
// ancestor (#3737): the inner RecyclerView on Android Settings claims hidden content above even at
// the top, while the outer ScrollView framing the same pixels reports where the content is.
// ---------------------------------------------------------------------------

test('scroll top edge: the Android Settings list at the top reports no hidden content above', async () => {
const state = await capture(settingsHomeTree('at-top'), 'top');
assert.equal(state.canScroll, false);
});

test('scroll top on Android Settings already at the top costs no pass under scoped recapture', async () => {
// Scoped captures return only the scope root's subtree, as an Android scoped snapshot does.
const tree = settingsHomeTree('at-top');
const outcome = await runScrollEdgePasses({
edge: 'top',
captureState: async (scope) => await capture(subtreeFor(tree, scope), 'top'),
scroll: async () => {
throw new Error('scroll top dispatched a gesture at the top');
},
});
assert.equal(outcome.passes, 0);
});

test('scroll top edge: a cyclic chain of same-rect scrollables ends the ancestor walk', async () => {
const rect = { x: 0, y: 100, width: 400, height: 600 };
const nodes = [
windowRoot(),
scrollNode(1, { parentIndex: 2, identifier: 'first', hiddenContentAbove: true, rect }),
scrollNode(2, { parentIndex: 1, identifier: 'second', rect }),
];
const state = await capture(nodes, 'top');
assert.equal(state.scope, 'first');
assert.equal(state.canScroll, true);
});

test('scroll top edge: the Android Settings list scrolled down still reports hidden content above', async () => {
const state = await capture(settingsHomeTree('scrolled-down'), 'top');
assert.equal(state.canScroll, true);
});

test('scroll bottom edge: the Android Settings list at the top still reports hidden content below', async () => {
const state = await capture(settingsHomeTree('at-top'), 'bottom');
assert.equal(state.canScroll, true);
});

test('scroll top on Android Settings stops at the top instead of reporting no progress', async () => {
const frames = [settingsHomeTree('scrolled-down'), settingsHomeTree('at-top')];
let scrolls = 0;
const outcome = await runScrollEdgePasses({
edge: 'top',
captureState: async () => await capture(frames[Math.min(scrolls, frames.length - 1)]!, 'top'),
scroll: async () => {
scrolls += 1;
},
});
assert.equal(outcome.passes, 1);
});

test('scroll top edge: a nested scroller whose rect differs from its scrollable ancestor keeps its own hint', async () => {
const nodes = [
windowRoot(),
scrollNode(1, { identifier: 'outer', rect: { x: 0, y: 0, width: 400, height: 800 } }),
scrollNode(2, {
parentIndex: 1,
identifier: 'inner',
hiddenContentAbove: true,
rect: { x: 0, y: 100, width: 400, height: 600 },
}),
];
const state = await capture(nodes, 'top');
assert.equal(state.scope, 'inner');
assert.equal(state.canScroll, true);
});

test('scroll top edge: a targeted node in a cyclic chain with no scrollable ancestor ends the walk', async () => {
const nodes = [
windowRoot(),
scrollNode(1, { identifier: 'list', hiddenContentAbove: true }),
{ ...scrollNode(2, { parentIndex: 3 }), type: 'Button', kind: 'button' },
{ ...scrollNode(3, { parentIndex: 2 }), type: 'Group', kind: 'group' },
];
const state = await capture(nodes, 'top', { nodeIndex: 2 });
assert.equal(state.scope, 'list');
assert.equal(state.canScroll, true);
});

test('scroll bottom edge: a same-rect horizontal pager does not stand in for the vertical list on its page', async () => {
// A horizontal pager reports no vertical edge hints; its page's vertical list carries them.
const rect = { x: 0, y: 100, width: 400, height: 600 };
const nodes = [
windowRoot(),
scrollNode(1, { identifier: 'pager', rect }),
scrollNode(2, { parentIndex: 1, identifier: 'feed', hiddenContentBelow: true, rect }),
];
const state = await capture(nodes, 'bottom');
assert.equal(state.scope, 'feed');
assert.equal(state.canScroll, true);
});
69 changes: 65 additions & 4 deletions packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,23 @@ function selectScrollContainer(
target: ScrollEdgeTarget,
): SnapshotNode | null {
const visibility = createSnapshotVisibility(nodes);
const scrollables = nodes.filter((node) => isScrollableNodeLike(node) && isUsableRect(node.rect));
const reportsVerticalEdge = (node: SnapshotNode): boolean =>
hasHiddenContentAtEdge(node, hiddenHints.get(node.index), 'top') ||
hasHiddenContentAtEdge(node, hiddenHints.get(node.index), 'bottom');
const edgeContainer = (node: SnapshotNode): SnapshotNode =>
sameRectScrollableAncestor(node, visibility.nodeByIndex, reportsVerticalEdge) ?? node;
const scrollables = [
...new Set(
nodes
.filter((node) => isScrollableNodeLike(node) && isUsableRect(node.rect))
.map(edgeContainer),
),
];
if (scrollables.length === 0) return null;

const targetAncestor = findNearestScrollableAncestor(target.nodeIndex, visibility.nodeByIndex);
if (targetAncestor) {
return targetAncestor;
return edgeContainer(targetAncestor);
Comment thread
airowe marked this conversation as resolved.
}

const targetPoint = target.point;
Expand Down Expand Up @@ -136,12 +147,14 @@ function findNearestScrollableAncestor(
byIndex: ReadonlyMap<number, SnapshotNode>,
): SnapshotNode | null {
if (nodeIndex === undefined) return null;
const visited = new Set<number>();
let node = byIndex.get(nodeIndex);
while (node) {
while (node && !visited.has(node.index)) {
visited.add(node.index);
if (isScrollableNodeLike(node) && isUsableRect(node.rect)) {
return node;
}
node = node.parentIndex === undefined ? undefined : byIndex.get(node.parentIndex);
node = parentOf(node, byIndex);
}
return null;
}
Expand All @@ -157,6 +170,54 @@ function hasHiddenContentAtEdge(
return node.hiddenContentAbove === true || hint?.hiddenContentAbove === true;
}

/**
* The outermost scrollable in an unbroken chain of same-rect, vertically scrolling ancestors, or
* undefined when the nearest scrollable ancestor frames different pixels or reports no vertical edge.
* Scroll edge selection stands that ancestor in for the nested scroller: Android's Settings
* RecyclerView offers scroll-backward even at the top, while the ScrollView framing the same pixels
* reports the real position (#3737). Choosing the ancestor also keeps the nested scroller inside the
* scoped recapture. Visited indexes stop a cyclic `parentIndex` chain in a malformed tree.
*
* An identical rect is a geometric heuristic for "these two scrollers move the same content", and
* geometry carries no scroll axis. Requiring the ancestor to report a vertical edge hint keeps a
* horizontal pager whose page holds a vertical list from standing in for that list; the snapshot
* emits vertical edge hints only for vertical scrollers. A vertical ancestor whose content fits in
* both directions reports no edge either, and leaves the nested scroller to answer for itself.
*/
function sameRectScrollableAncestor(
node: SnapshotNode,
byIndex: ReadonlyMap<number, SnapshotNode>,
reportsVerticalEdge: (candidate: SnapshotNode) => boolean,
): SnapshotNode | undefined {
const visited = new Set<number>([node.index]);
let owner: SnapshotNode | undefined;
let current = node;
let parent = parentOf(current, byIndex);
while (parent && !visited.has(parent.index)) {
visited.add(parent.index);
if (isScrollableNodeLike(parent) && isUsableRect(parent.rect)) {
if (!sameRect(parent.rect, current.rect) || !reportsVerticalEdge(parent)) return owner;
owner = parent;
current = parent;
}
parent = parentOf(parent, byIndex);
}
return owner;
}

function parentOf(
node: SnapshotNode,
byIndex: ReadonlyMap<number, SnapshotNode>,
): SnapshotNode | undefined {
return node.parentIndex === undefined ? undefined : byIndex.get(node.parentIndex);
}

function sameRect(a: SnapshotNode['rect'], b: SnapshotNode['rect']): boolean {
return Boolean(
a && b && a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height,
);
}

function buildScrollContainerScope(
node: SnapshotNode,
nodes: readonly SnapshotNode[],
Expand Down
Loading