fix(scroll): keep adb-backed Android directional swipes inside the resolved scrollable

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- On adb-backed Android, a directional `scroll` keeps both swipe endpoints inside the scroll container the current snapshot resolves, inset from its edges and within the 10% edge padding, so a list under a header or above a footer no longer receives a drag that starts outside it and ends in `scroll_no_progress`. A swipe that already fits is unchanged; one longer than the container is shortened, and the response reports the dispatched `pixels` beside `requestedPixels`. Horizontal scrolls, scrolls after a mutation that has not been re-snapshotted, iOS, and WebDriver (Appium) sessions keep the unclamped swipe for now (#2714).
 - Add `uninstall <app>` and `client.apps.uninstall({ app })` to remove an installed app on adb-backed Android targets, iOS simulators, and CoreDevice iOS devices. Both report `wasInstalled`. A package name or bundle id that is not installed succeeds with `wasInstalled: false`. Other targets, Appium and Limrun sessions included, refuse it before anything runs.
 - Add `settings mobile-data on|off` for adb-backed Android targets. It switches mobile data through `svc data` and reports the `mobileData` state stored in the global `mobile_data` row; multi-SIM read-back is not covered yet. Every other target refuses it before anything runs, and `--app` is refused because the switch is device-wide.
 - On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect.

packages/contracts/src/scroll-command.ts

@@ -1,6 +1,7 @@
 import { AppError } from '@agent-device/kernel/errors';
 import type { SettleObservation } from './interaction.ts';
 import type { ScrollDirection } from './scroll-gesture.ts';
+import type { Rect } from '@agent-device/kernel/snapshot';
 
 export const SCROLL_DURATION_MAX_MS = 10_000;
 export const DEFAULT_MOBILE_SCROLL_DURATION_MS = 300;
@@ -38,13 +39,24 @@ export type ScrollTimingOptions = {
 
 export type ScrollCommandOptions = ScrollDistanceOptions & ScrollTimingOptions;
 
-export type ScrollExecutionOptions = ScrollCommandOptions & {
-  releaseBehavior?: ScrollReleaseBehavior;
+/**
+ * The scroll container the caller resolved from the current tree, in the snapshot's absolute space.
+ * An owner that plans its own swipe keeps both endpoints inside it (#2714); an owner without that
+ * planner ignores it.
+ */
+export type ScrollContainerTarget = {
+  containerRect?: Rect;
 };
 
-export type ResolvedScrollExecutionOptions = ScrollCommandOptions & {
-  releaseBehavior: ScrollReleaseBehavior;
-};
+export type ScrollExecutionOptions = ScrollCommandOptions &
+  ScrollContainerTarget & {
+    releaseBehavior?: ScrollReleaseBehavior;
+  };
+
+export type ResolvedScrollExecutionOptions = ScrollCommandOptions &
+  ScrollContainerTarget & {
+    releaseBehavior: ScrollReleaseBehavior;
+  };
 
 export function resolveScrollExecutionOptions(
   options: ScrollCommandOptions,

packages/contracts/src/scroll-gesture-container.test.ts

@@ -0,0 +1,179 @@
+import { test } from 'vitest';
+import assert from 'node:assert/strict';
+import type { Rect } from '@agent-device/kernel/snapshot';
+import { buildScrollGesturePlan, type ScrollDirection } from './scroll-gesture.ts';
+
+// The iPhone geometry #2714 was reported on: a 402x874 frame, the default 10% edge padding leaving
+// x 40..362 and y 87..787 for a swipe.
+const FRAME = { referenceWidth: 402, referenceHeight: 874 };
+const PADDED_LEFT = 40;
+const PADDED_TOP = 87;
+const PADDED_BOTTOM = 787;
+// Inset under a navigation bar and running to the bottom of the frame.
+const INSET_CONTAINER: Rect = { x: 18, y: 178, width: 366, height: 662 };
+// Inset at both ends: a footer sits below it, well above the padded band's bottom.
+const FOOTERED_CONTAINER: Rect = { x: 18, y: 178, width: 366, height: 500 };
+const FITTING_CONTAINER: Rect = { x: 18, y: 62, width: 366, height: 729 };
+
+type Endpoints = { x1: number; y1: number; x2: number; y2: number };
+
+/**
+ * A container rect is half-open, as Android bounds and View hit-testing are: `y + height` is the
+ * first row outside it. Strictly inside on both sides also keeps the drag off the edge row.
+ */
+function strictlyInside(plan: Endpoints, rect: Rect): boolean {
+  const inside = (x: number, y: number) =>
+    x > rect.x && x < rect.x + rect.width && y > rect.y && y < rect.y + rect.height;
+  return inside(plan.x1, plan.y1) && inside(plan.x2, plan.y2);
+}
+
+test.each<[ScrollDirection, string, Rect]>([
+  ['up', 'inset', INSET_CONTAINER],
+  ['down', 'inset', INSET_CONTAINER],
+  ['up', 'footered', FOOTERED_CONTAINER],
+  ['down', 'footered', FOOTERED_CONTAINER],
+])(
+  'a %s swipe longer than an %s container shrinks inside it and reports both spans',
+  (direction, _, container) => {
+    const unclamped = buildScrollGesturePlan({ direction, amount: 0.75, ...FRAME });
+    const plan = buildScrollGesturePlan({ direction, amount: 0.75, ...FRAME, container });
+
+    assert.ok(strictlyInside(plan, container), JSON.stringify(plan));
+    assert.ok(Math.max(plan.y1, plan.y2) <= PADDED_BOTTOM, JSON.stringify(plan));
+    assert.equal(plan.pixels, Math.abs(plan.y2 - plan.y1));
+    assert.ok(plan.pixels < unclamped.pixels);
+    assert.equal(plan.requestedPixels, unclamped.pixels);
+    assert.equal(Math.sign(plan.y2 - plan.y1), Math.sign(unclamped.y2 - unclamped.y1));
+  },
+);
+
+test.each<ScrollDirection>(['left', 'right'])(
+  'a %s swipe longer than a narrow container shrinks inside it along x',
+  (direction) => {
+    const column: Rect = { x: 100, y: 0, width: 200, height: 874 };
+    const unclamped = buildScrollGesturePlan({ direction, ...FRAME });
+    const plan = buildScrollGesturePlan({ direction, ...FRAME, container: column });
+
+    assert.ok(strictlyInside(plan, column), JSON.stringify(plan));
+    assert.deepEqual([plan.y1, plan.y2], [unclamped.y1, unclamped.y2]);
+    assert.equal(plan.pixels, Math.abs(plan.x2 - plan.x1));
+    assert.ok(plan.pixels < unclamped.pixels);
+    assert.equal(plan.requestedPixels, unclamped.pixels);
+    assert.equal(Math.sign(plan.x2 - plan.x1), Math.sign(unclamped.x2 - unclamped.x1));
+  },
+);
+
+test('a horizontal swipe that fits an offset container moves into it without shortening', () => {
+  const rightPane: Rect = { x: 180, y: 0, width: 222, height: 874 };
+  const unclamped = buildScrollGesturePlan({ direction: 'left', pixels: 100, ...FRAME });
+  const plan = buildScrollGesturePlan({
+    direction: 'left',
+    pixels: 100,
+    ...FRAME,
+    container: rightPane,
+  });
+
+  assert.ok(strictlyInside(plan, rightPane), JSON.stringify(plan));
+  assert.equal(Math.abs(plan.x2 - plan.x1), Math.abs(unclamped.x2 - unclamped.x1));
+  assert.equal(plan.pixels, unclamped.pixels);
+  assert.equal('requestedPixels' in plan, false);
+});
+
+test.each<ScrollDirection>(['up', 'down', 'left', 'right'])(
+  'a %s swipe already inside its container keeps today’s endpoints and distance',
+  (direction) => {
+    const unclamped = buildScrollGesturePlan({ direction, amount: 0.75, ...FRAME });
+    const plan = buildScrollGesturePlan({
+      direction,
+      amount: 0.75,
+      ...FRAME,
+      container: FITTING_CONTAINER,
+    });
+
+    assert.deepEqual(plan, unclamped);
+    assert.equal('requestedPixels' in plan, false);
+  },
+);
+
+test('a swipe that fits an offset container moves into it without shortening', () => {
+  const lowerContainer: Rect = { x: 0, y: 300, width: 402, height: 574 };
+  const unclamped = buildScrollGesturePlan({ direction: 'down', amount: 0.5, ...FRAME });
+  const plan = buildScrollGesturePlan({
+    direction: 'down',
+    amount: 0.5,
+    ...FRAME,
+    container: lowerContainer,
+  });
+
+  assert.ok(strictlyInside(plan, lowerContainer), JSON.stringify(plan));
+  assert.equal(Math.abs(plan.y1 - plan.y2), Math.abs(unclamped.y1 - unclamped.y2));
+  assert.equal(plan.pixels, unclamped.pixels);
+  assert.equal('requestedPixels' in plan, false);
+});
+
+test('an odd span that only loses its rounding pixel is not reported as shortened', () => {
+  // On an 875-point axis the padded band is 88..787 (699 points), and a 699-point request centred
+  // on 438 rounds its halves to a 700-point lane. Fitting it into the band drops that rounding
+  // pixel, not any of the travel asked for.
+  const frame = { referenceWidth: 402, referenceHeight: 875 };
+  const wholeFrame: Rect = { x: 0, y: 0, width: 402, height: 875 };
+  const plan = buildScrollGesturePlan({
+    direction: 'up',
+    pixels: 699,
+    ...frame,
+    container: wholeFrame,
+  });
+
+  assert.equal(plan.pixels, 699);
+  assert.equal(Math.abs(plan.y2 - plan.y1), 699);
+  assert.equal('requestedPixels' in plan, false);
+});
+
+test('a vertical swipe beside a narrow container moves across into it', () => {
+  const rightColumn: Rect = { x: 250, y: 0, width: 152, height: 874 };
+  const unclamped = buildScrollGesturePlan({ direction: 'up', ...FRAME });
+  const plan = buildScrollGesturePlan({ direction: 'up', ...FRAME, container: rightColumn });
+
+  assert.ok(strictlyInside(plan, rightColumn), JSON.stringify(plan));
+  assert.ok(plan.x1 <= FRAME.referenceWidth - PADDED_LEFT, `swipe x ${plan.x1} is in the padding`);
+  assert.equal(plan.x2, plan.x1);
+  assert.deepEqual([plan.y1, plan.y2, plan.pixels], [unclamped.y1, unclamped.y2, unclamped.pixels]);
+});
+
+test('a container reaching into the side padding is swiped in the part the padding leaves', () => {
+  const edgeColumn: Rect = { x: 0, y: 0, width: 60, height: 874 };
+  const plan = buildScrollGesturePlan({ direction: 'up', ...FRAME, container: edgeColumn });
+
+  assert.ok(strictlyInside(plan, edgeColumn), JSON.stringify(plan));
+  assert.ok(plan.x1 >= PADDED_LEFT, `swipe x ${plan.x1} is inside the side padding`);
+});
+
+test.each<[ScrollDirection, string, Rect]>([
+  [
+    'up',
+    'a navigation rail along the left edge',
+    { x: 0, y: 0, width: PADDED_LEFT - 1, height: 874 },
+  ],
+  [
+    'left',
+    'a chip row just above the padded band',
+    { x: 0, y: 60, width: 402, height: PADDED_TOP - 61 },
+  ],
+])(
+  'a %s swipe aimed at %s runs inside it when the cross-axis padding leaves none of it',
+  (direction, _, container) => {
+    const unclamped = buildScrollGesturePlan({ direction, ...FRAME });
+    const plan = buildScrollGesturePlan({ direction, ...FRAME, container });
+
+    assert.ok(strictlyInside(plan, container), JSON.stringify(plan));
+    assert.equal(plan.pixels, unclamped.pixels);
+  },
+);
+
+test('a container lying wholly in the padding of the scrolled axis leaves the padded swipe as it was', () => {
+  const statusBand: Rect = { x: 0, y: 0, width: 402, height: PADDED_TOP - 1 };
+  const unclamped = buildScrollGesturePlan({ direction: 'up', ...FRAME });
+  const plan = buildScrollGesturePlan({ direction: 'up', ...FRAME, container: statusBand });
+
+  assert.deepEqual(plan, unclamped);
+});

packages/contracts/src/scroll-gesture.ts

@@ -49,6 +49,14 @@ export type ScrollGestureOptions = {
   pixels?: number;
   referenceWidth: number;
   referenceHeight: number;
+  /**
+   * The scroll container the swipe is meant for, in the reference frame's own zero-origin space.
+   * Both endpoints stay strictly inside its intersection with the padded frame (#2714): a span
+   * that already fits is unchanged, one that leaves it is moved in, and one longer than it is
+   * shortened. A container the padding leaves nothing of along the scrolled axis gets the padded
+   * swipe unchanged, since the safe bounds outrank the container.
+   */
+  container?: Rect;
 };
 
 export type ScrollGesturePlan = {
@@ -60,7 +68,10 @@ export type ScrollGesturePlan = {
   referenceWidth: number;
   referenceHeight: number;
   amount?: number;
+  /** The span dispatched, which is less than `requestedPixels` when the container shortened it. */
   pixels: number;
+  /** Present only when the container shortened the swipe: the span the padded frame would carry. */
+  requestedPixels?: number;
 };
 
 export type SwipePresetGesturePlan = {
@@ -97,44 +108,122 @@ export const DEFAULT_SCROLL_AMOUNT = 0.6;
 const DEFAULT_EDGE_PADDING_FRACTION = 0.1;
 // Edge presets stay close to the system gesture boundary without emitting edge coordinates.
 const SWIPE_PRESET_EDGE_MARGIN_PX = 8;
+// A swipe kept inside a container stays this far off its edges, so the drag never starts on the
+// container's boundary row.
+const SCROLL_CONTAINER_EDGE_INSET = 8;
 
 export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGesturePlan {
   const direction = options.direction;
-  const axisLength =
-    direction === 'up' || direction === 'down' ? options.referenceHeight : options.referenceWidth;
+  const vertical = direction === 'up' || direction === 'down';
+  const axisLength = vertical ? options.referenceHeight : options.referenceWidth;
+  const crossLength = vertical ? options.referenceWidth : options.referenceHeight;
   const requestedAmount = resolveRequestedAmount(options.amount);
   const requestedPixels =
     options.pixels !== undefined
       ? normalizeRequestedPixels(options.pixels)
       : Math.round(axisLength * requestedAmount);
-  const edgePadding = Math.max(1, Math.round(axisLength * DEFAULT_EDGE_PADDING_FRACTION));
+  const edgePadding = scrollEdgePadding(axisLength);
   const maxTravelPixels = Math.max(1, axisLength - edgePadding * 2);
   const travelPixels = Math.max(1, Math.min(requestedPixels, maxTravelPixels));
   const halfTravel = Math.round(travelPixels / 2);
-  const centerX = Math.round(options.referenceWidth / 2);
-  const centerY = Math.round(options.referenceHeight / 2);
-  const buildPlan = (x1: number, y1: number, x2: number, y2: number): ScrollGesturePlan => ({
+  const axisCenter = Math.round(axisLength / 2);
+  const unclamped = {
+    axisStart: axisCenter - halfTravel,
+    axisEnd: axisCenter + halfTravel,
+    cross: Math.round(crossLength / 2),
+  };
+  const crossPadding = scrollEdgePadding(crossLength);
+  const lane = options.container
+    ? keepSwipeInContainer(unclamped, options.container, vertical, {
+        axis: { min: edgePadding, max: axisLength - edgePadding },
+        cross: { min: crossPadding, max: crossLength - crossPadding },
+        crossFrame: { min: 0, max: crossLength },
+      })
+    : unclamped;
+  const laneLength = lane.axisEnd - lane.axisStart;
+  // Centring rounds an odd span up by one, so only a lane shorter than the travel itself is short.
+  const shortened = laneLength < travelPixels;
+  return {
     direction,
-    x1,
-    y1,
-    x2,
-    y2,
+    ...laneEndpoints(direction, lane),
     referenceWidth: options.referenceWidth,
     referenceHeight: options.referenceHeight,
     amount: options.amount,
-    pixels: travelPixels,
-  });
+    pixels: shortened ? laneLength : travelPixels,
+    ...(shortened ? { requestedPixels: travelPixels } : {}),
+  };
+}
 
-  switch (direction) {
-    case 'up':
-      return buildPlan(centerX, centerY - halfTravel, centerX, centerY + halfTravel);
-    case 'down':
-      return buildPlan(centerX, centerY + halfTravel, centerX, centerY - halfTravel);
-    case 'left':
-      return buildPlan(centerX - halfTravel, centerY, centerX + halfTravel, centerY);
-    case 'right':
-      return buildPlan(centerX + halfTravel, centerY, centerX - halfTravel, centerY);
-  }
+/** `up`/`left` drag from the lane's low end toward its high end; `down`/`right` drag the other way. */
+function laneEndpoints(
+  direction: ScrollDirection,
+  lane: SwipeLane,
+): Pick<ScrollGesturePlan, 'x1' | 'y1' | 'x2' | 'y2'> {
+  const vertical = direction === 'up' || direction === 'down';
+  const low = vertical
+    ? { x: lane.cross, y: lane.axisStart }
+    : { x: lane.axisStart, y: lane.cross };
+  const high = vertical ? { x: lane.cross, y: lane.axisEnd } : { x: lane.axisEnd, y: lane.cross };
+  const [start, end] = direction === 'up' || direction === 'left' ? [low, high] : [high, low];
+  return { x1: start.x, y1: start.y, x2: end.x, y2: end.y };
+}
+
+type SwipeLane = { axisStart: number; axisEnd: number; cross: number };
+
+type CoordinateRange = { min: number; max: number };
+
+function scrollEdgePadding(length: number): number {
+  return Math.max(1, Math.round(length * DEFAULT_EDGE_PADDING_FRACTION));
+}
+
+/**
+ * Moves a centred swipe lane inside the container's intersection with the padded frame,
+ * shortening it only when that intersection is shorter than the lane. The container is half-open,
+ * as Android bounds and View hit-testing are, and inset from its edges, so every endpoint lands on
+ * a row the container owns. A container the padding leaves nothing of on the scrolled axis keeps
+ * the lane the padded frame planned: the safe bounds outrank the container, and the movement read
+ * reports the miss as `container-outside-swipe` when nothing else moved.
+ */
+function keepSwipeInContainer(
+  lane: SwipeLane,
+  container: Rect,
+  vertical: boolean,
+  padded: { axis: CoordinateRange; cross: CoordinateRange; crossFrame: CoordinateRange },
+): SwipeLane {
+  const band = insetContainerRange(
+    vertical ? container.y : container.x,
+    vertical ? container.height : container.width,
+    padded.axis,
+  );
+  const crossStart = vertical ? container.x : container.y;
+  const crossSize = vertical ? container.width : container.height;
+  // The padding guards the scrolled axis against system bars. Across it, a rail or chip row that
+  // sits wholly in the padding is still the surface asked for, so the swipe runs along it there.
+  const crossBand =
+    insetContainerRange(crossStart, crossSize, padded.cross) ??
+    insetContainerRange(crossStart, crossSize, padded.crossFrame);
+  if (!band || !crossBand) return lane;
+
+  const cross =
+    lane.cross >= crossBand.min && lane.cross <= crossBand.max
+      ? lane.cross
+      : Math.round((crossBand.min + crossBand.max) / 2);
+  const length = lane.axisEnd - lane.axisStart;
+  if (length >= band.max - band.min) return { axisStart: band.min, axisEnd: band.max, cross };
+  const shift =
+    lane.axisStart < band.min ? band.min - lane.axisStart : Math.min(0, band.max - lane.axisEnd);
+  return { axisStart: lane.axisStart + shift, axisEnd: lane.axisEnd + shift, cross };
+}
+
+/** The coordinates of `[start, start + size)` an inset swipe may use inside `padded`, if any. */
+function insetContainerRange(
+  start: number,
+  size: number,
+  padded: CoordinateRange,
+): CoordinateRange | undefined {
+  const min = Math.max(padded.min, Math.ceil(start) + SCROLL_CONTAINER_EDGE_INSET);
+  const max = Math.min(padded.max, Math.floor(start + size) - 1 - SCROLL_CONTAINER_EDGE_INSET);
+  return max > min ? { min, max } : undefined;
 }
 
 /**