From b739edfe08ab9a031341be6f9790784df30dcc00 Mon Sep 17 00:00:00 2001 From: JJ Date: Mon, 21 Sep 2026 18:03:36 +0800 Subject: [PATCH 1/3] fix(expo): restore annotation gestures inside the wizard modal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing could be drawn in the Annotate step on Android: strokes were ignored while the toolbar kept working, so it read as a broken canvas. iOS was fine. A react-native Modal on Android is a separate native window (ReactModalHostView.DialogRootViewGroup) that sits outside the host app's GestureHandlerRootView, so gesture-handler receives no touches inside it and the pan gesture never fires. Gesture-handler's own root view detects this case and activates a nested root for it, so the wizard has to carry its own. The launcher is unaffected because it renders in the app's own tree, which is why tapping the bug button still opened the wizard. Tests need react-native mounted under bun, which means stubbing the native modules; that harness lands in src/test-support/native-mocks.ts. bun test shares one module registry across files, so system-info.test.ts now takes react-native from the same stub instead of registering a partial one that decided what the component tests could import. test-sdk.sh grew *.test.tsx discovery — its find only matched *.test.ts, so component tests would never have run in CI. Co-Authored-By: Claude Opus 5 (1M context) --- bun.lock | 18 +- docs/guide/expo.md | 1 + packages/expo/package.json | 2 + .../expo/src/collectors/system-info.test.ts | 14 +- .../expo/src/test-support/native-mocks.ts | 141 ++++++++++ .../expo/src/wizard/sheet.gestures.test.tsx | 72 +++++ packages/expo/src/wizard/sheet.tsx | 245 +++++++++--------- scripts/test-sdk.sh | 5 +- 8 files changed, 366 insertions(+), 132 deletions(-) create mode 100644 packages/expo/src/test-support/native-mocks.ts create mode 100644 packages/expo/src/wizard/sheet.gestures.test.tsx diff --git a/bun.lock b/bun.lock index 62747107..97dc99e8 100644 --- a/bun.lock +++ b/bun.lock @@ -69,7 +69,7 @@ }, "apps/extension": { "name": "@reprojs/extension", - "version": "0.1.4", + "version": "0.2.2", "dependencies": { "preact": "^10.23.0", }, @@ -87,7 +87,7 @@ }, "packages/core": { "name": "@reprojs/core", - "version": "0.4.2", + "version": "0.5.2", "devDependencies": { "@preact/signals": "^1.3.0", "@reprojs/recorder": "workspace:*", @@ -104,7 +104,7 @@ }, "packages/expo": { "name": "@reprojs/expo", - "version": "0.3.2", + "version": "0.3.4", "dependencies": { "@expo/config-plugins": "^8.0.0", }, @@ -112,10 +112,12 @@ "@reprojs/sdk-utils": "workspace:*", "@reprojs/shared": "workspace:*", "@types/react": "^18.3.3", + "@types/react-test-renderer": "18", "bun-types": "^1.3.12", "expo": "^52.0.0", "react": "^18.3.1", "react-native": "^0.76.0", + "react-test-renderer": "18.3.1", "tsdown": "^0.21.9", "typescript": "^5.8", }, @@ -1833,7 +1835,7 @@ "@types/react": ["@types/react@18.3.28", "", { "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" } }, "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw=="], - "@types/react-test-renderer": ["@types/react-test-renderer@19.1.0", "", { "dependencies": { "@types/react": "*" } }, "sha512-XD0WZrHqjNrxA/MaR9O22w/RNidWR9YZmBdRGI7wcnWGrv/3dA8wKCJ8m63Sn+tLJhcjmuhOi629N66W6kgWzQ=="], + "@types/react-test-renderer": ["@types/react-test-renderer@18.3.1", "", { "dependencies": { "@types/react": "^18" } }, "sha512-vAhnk0tG2eGa37lkU9+s5SoroCsRI08xnsWFiAXOuPH2jqzMbcXvKExXViPi1P5fIklDeCvXqyrdmipFaSkZrA=="], "@types/resolve": ["@types/resolve@1.20.2", "", {}, "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q=="], @@ -3481,6 +3483,10 @@ "react-refresh": ["react-refresh@0.14.2", "", {}, "sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA=="], + "react-shallow-renderer": ["react-shallow-renderer@16.15.0", "", { "dependencies": { "object-assign": "^4.1.1", "react-is": "^16.12.0 || ^17.0.0 || ^18.0.0" }, "peerDependencies": { "react": "^16.0.0 || ^17.0.0 || ^18.0.0" } }, "sha512-oScf2FqQ9LFVQgA73vr86xl2NaOIX73rh+YFqcOp68CWj56tSfgtGKrEbyhCj0rSijyG9M1CYprTh39fBi5hzA=="], + + "react-test-renderer": ["react-test-renderer@18.3.1", "", { "dependencies": { "react-is": "^18.3.1", "react-shallow-renderer": "^16.15.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-KkAgygexHUkQqtvvx/otwxtuFu5cVjfzTCtjXLH9boS19/Nbtg84zS7wIQn39G8IlrhThBpQsMKkq5ZHZIYFXA=="], + "readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], "readdir-glob": ["readdir-glob@1.1.3", "", { "dependencies": { "minimatch": "^5.1.0" } }, "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA=="], @@ -4709,6 +4715,10 @@ "react-devtools-core/ws": ["ws@7.5.10", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": "^5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ=="], + "react-native-gesture-handler/@types/react-test-renderer": ["@types/react-test-renderer@19.1.0", "", { "dependencies": { "@types/react": "*" } }, "sha512-XD0WZrHqjNrxA/MaR9O22w/RNidWR9YZmBdRGI7wcnWGrv/3dA8wKCJ8m63Sn+tLJhcjmuhOi629N66W6kgWzQ=="], + + "react-test-renderer/scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + "readable-stream/buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], "readdir-glob/minimatch": ["minimatch@5.1.9", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw=="], diff --git a/docs/guide/expo.md b/docs/guide/expo.md index 4d82eb0f..59746067 100644 --- a/docs/guide/expo.md +++ b/docs/guide/expo.md @@ -189,6 +189,7 @@ The queue is not encrypted — documented privacy tradeoff for v1. Don't use as - **`Submission too fast` (400)** — the server's dwell gate (default 1000 ms) didn't pass. The SDK clamps to ≥1000 ms so this is only seen if you're running an older SDK build. - **Wizard opens but screenshot area is blank** — `react-native-view-shot` returned without an error but produced a black frame. Usually resolved by dismissing the keyboard before opening the wizard. - **Annotations don't appear in the submitted PNG** — older SDK bug; update to ≥0.1.0. +- **Nothing draws on the screenshot in the Annotate step (Android only)** — fixed in 0.3.5, upgrade. On Android a react-native `Modal` is a separate native window that sits outside your app's `GestureHandlerRootView`, so gesture-handler received no touches inside the wizard and the canvas ignored every stroke. The toolbar still responded, which makes this look like a broken canvas rather than a dead gesture. iOS was unaffected. - **`xcodebuild: error: Unable to find a destination matching the provided destination specifier` / `iOS is not installed`** — Xcode is installed but its iOS platform component isn't, so *no* iOS destination is eligible (not even a booted simulator). Install it via Xcode → Settings → Components, or run `xcodebuild -downloadPlatform iOS`. Confirm with `xcodebuild -workspace ios/.xcworkspace -scheme -showdestinations` — if every entry is listed as "Ineligible", the platform is the missing piece, not your config. - **`run:android` fails with no emulator** — `emulator -list-avds` printing nothing means no AVD exists. Create one in Android Studio → Device Manager (this downloads a system image), or plug in a physical device with USB debugging on. - **Report never arrives and the app logs a network error** — you're almost certainly pointing at the wrong host. Re-check [Reaching the dashboard from a device](#reaching-the-dashboard-from-a-device): an Android emulator cannot resolve `localhost` to your laptop, it needs `10.0.2.2`. diff --git a/packages/expo/package.json b/packages/expo/package.json index c27b40a1..4043496f 100644 --- a/packages/expo/package.json +++ b/packages/expo/package.json @@ -48,10 +48,12 @@ "@reprojs/sdk-utils": "workspace:*", "@reprojs/shared": "workspace:*", "@types/react": "^18.3.3", + "@types/react-test-renderer": "18", "bun-types": "^1.3.12", "expo": "^52.0.0", "react": "^18.3.1", "react-native": "^0.76.0", + "react-test-renderer": "18.3.1", "tsdown": "^0.21.9", "typescript": "^5.8" }, diff --git a/packages/expo/src/collectors/system-info.test.ts b/packages/expo/src/collectors/system-info.test.ts index 54ffc672..840b1d69 100644 --- a/packages/expo/src/collectors/system-info.test.ts +++ b/packages/expo/src/collectors/system-info.test.ts @@ -1,14 +1,10 @@ import { test, expect, mock } from "bun:test" +import { installNativeMocks } from "../test-support/native-mocks" -// Mock the native modules BEFORE importing the collector. -mock.module("react-native", () => ({ - Platform: { OS: "ios", Version: "17.4" }, - Dimensions: { - get: (k: "window" | "screen") => - k === "window" ? { width: 390, height: 844 } : { width: 1179, height: 2556 }, - }, - PixelRatio: { get: () => 3 }, -})) +// Mock the native modules BEFORE importing the collector. react-native comes +// from the shared stub so this file and the component tests agree on one +// surface — see the note in native-mocks.ts. +installNativeMocks() mock.module("expo-device", () => ({ modelName: "iPhone 15", })) diff --git a/packages/expo/src/test-support/native-mocks.ts b/packages/expo/src/test-support/native-mocks.ts new file mode 100644 index 00000000..e6ef4b13 --- /dev/null +++ b/packages/expo/src/test-support/native-mocks.ts @@ -0,0 +1,141 @@ +import { mock } from "bun:test" +import type React from "react" + +/** + * Replaces every native module the wizard tree touches with host-component + * stand-ins, so `react-test-renderer` can mount it under `bun test`. Each + * stub is the *string* name the node will carry in the rendered tree, which + * is what the structural assertions match on. + * + * `react-native-svg`'s `Text` is deliberately stubbed as `"SvgText"` so a tree + * assertion can tell an SVG label apart from a react-native ``. + * + * Call this before importing the module under test — `mock.module` only + * affects imports that happen after it runs. + * + * Every test in this package that needs `react-native` must go through here. + * `bun test` shares one module registry across files, so a test file that + * registers its own partial `react-native` stub decides what *other* files + * see: components then import a `View` that does not exist, bun falls through + * to the real Flow-typed source, and the suite fails only when run together. + */ +export const RN_DEVICE = { + os: "ios", + osVersion: "17.4", + window: { width: 390, height: 844 }, + screen: { width: 1179, height: 2556 }, + pixelRatio: 3, +} as const + +function pickIos(options: Record): unknown { + return options.ios +} + +export function installNativeMocks(): void { + mock.module("react-native", () => ({ + ActivityIndicator: "ActivityIndicator", + Image: "Image", + KeyboardAvoidingView: "KeyboardAvoidingView", + Modal: "Modal", + Pressable: "Pressable", + SafeAreaView: "SafeAreaView", + ScrollView: "ScrollView", + Text: "Text", + TextInput: "TextInput", + View: "View", + Platform: { OS: RN_DEVICE.os, Version: RN_DEVICE.osVersion, select: pickIos }, + Dimensions: { + get: (key: "window" | "screen") => + key === "window" ? { ...RN_DEVICE.window } : { ...RN_DEVICE.screen }, + }, + PixelRatio: { get: () => RN_DEVICE.pixelRatio }, + })) + + mock.module("react-native-svg", () => ({ + default: "Svg", + Path: "Path", + Rect: "Rect", + Text: "SvgText", + })) + + mock.module("react-native-gesture-handler", () => ({ + GestureHandlerRootView: "GestureHandlerRootView", + GestureDetector: "GestureDetector", + Gesture: { + Pan: () => gestureStub("pan"), + Tap: () => gestureStub("tap"), + Race: (...members: GestureStub[]) => gestureStub("race", members), + }, + })) + + mock.module("react-native-view-shot", () => ({ + captureRef: async () => "file:///flattened.png", + })) + + mock.module("expo-document-picker", () => ({ + getDocumentAsync: async () => ({ canceled: true, assets: null }), + })) + + mock.module("expo-image-picker", () => ({ + launchImageLibraryAsync: async () => ({ canceled: true, assets: null }), + MediaTypeOptions: { All: "All" }, + })) +} + +export type GestureKind = "pan" | "tap" | "race" + +export interface GestureStub { + kind: GestureKind + /** Callbacks the component registered, keyed by builder method name. */ + handlers: Record void> + /** Composed gestures, for `Gesture.Race(...)`. */ + members: GestureStub[] +} + +/** + * Stands in for a gesture builder. An unknown property access returns a + * function that returns the stub again, so any chain the canvas writes + * (`.runOnJS(true).minDistance(2).onStart(fn)`) resolves without the stub + * having to know which methods exist — while `onX` callbacks are recorded so a + * test can fire the gesture by hand. + */ +function gestureStub(kind: GestureKind, members: GestureStub[] = []): GestureStub { + const state: GestureStub = { kind, handlers: {}, members } + const proxy = new Proxy(state, { + get(target, prop) { + if (prop in target) return target[prop as keyof GestureStub] + return (arg: unknown) => { + if (typeof prop === "string" && prop.startsWith("on") && typeof arg === "function") { + target.handlers[prop] = arg as (event: unknown) => void + } + return proxy + } + }, + }) + return proxy +} + +/** Digs the gesture of the given kind out of whatever `GestureDetector` got. */ +export function gestureOfKind(gesture: unknown, kind: GestureKind): GestureStub { + const stub = gesture as GestureStub + if (stub.kind === kind) return stub + const found = stub.members.find((m) => m.kind === kind) + if (!found) throw new Error(`no ${kind} gesture found`) + return found +} + +/** Fires a recorded handler, or throws if the component never registered it. */ +export function fireGesture(stub: GestureStub, handler: string, event: unknown): void { + const fn = stub.handlers[handler] + if (!fn) throw new Error(`gesture has no ${handler} handler`) + fn(event) +} + +/** + * `findByType` typed for a mocked host component. The stubs above are plain + * strings, and react-test-renderer's signature only admits names React knows + * as JSX intrinsics — which native components never are. + */ +export function hostType(name: string): React.ElementType { + return name as unknown as React.ElementType +} diff --git a/packages/expo/src/wizard/sheet.gestures.test.tsx b/packages/expo/src/wizard/sheet.gestures.test.tsx new file mode 100644 index 00000000..9f90b822 --- /dev/null +++ b/packages/expo/src/wizard/sheet.gestures.test.tsx @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test" +import React from "react" +import { act, create, type ReactTestInstance } from "react-test-renderer" +import { hostType, installNativeMocks } from "../test-support/native-mocks" + +installNativeMocks() + +const SCREENSHOT = { uri: "file:///shot.png", width: 320, height: 640 } + +async function mountWizard() { + const { WizardSheet } = await import("./sheet") + let tree: ReturnType | null = null + act(() => { + tree = create( + undefined} + onClose={() => undefined} + />, + ) + }) + if (tree === null) throw new Error("wizard did not mount") + return tree as ReturnType +} + +/** Walks up from a node looking for an ancestor of the given host type. */ +function hasAncestorOfType(node: ReactTestInstance, type: string): boolean { + let current: ReactTestInstance | null = node.parent + while (current !== null) { + if (current.type === type) return true + current = current.parent + } + return false +} + +function advanceToAnnotateStep(tree: ReturnType) { + const primary = tree.root.find((n) => n.props.label === "Continue") + act(() => { + primary.props.onPress() + }) + // The canvas only mounts once the preview container has been measured, and + // react-test-renderer never fires layout on its own. + const measured = tree.root.find((n) => typeof n.props.onLayout === "function") + act(() => { + measured.props.onLayout({ nativeEvent: { layout: { width: 300, height: 500 } } }) + }) +} + +describe("WizardSheet gesture root", () => { + // Regression: on Android a react-native Modal is a separate native window + // (ReactModalHostView.DialogRootViewGroup) that sits outside the host app's + // GestureHandlerRootView, so gesture-handler never sees touches inside it + // and the annotation canvas could not be drawn on at all. iOS was fine, + // which is why this only showed up for Android reporters. + test("mounts a GestureHandlerRootView inside the Modal", async () => { + const tree = await mountWizard() + const modal = tree.root.findByType(hostType("Modal")) + const root = modal.findByType(hostType("GestureHandlerRootView")) + expect(root.findAllByType(hostType("SafeAreaView")).length).toBeGreaterThan(0) + }) + + test("annotation gestures are inside the gesture root", async () => { + const tree = await mountWizard() + advanceToAnnotateStep(tree) + const detectors = tree.root.findAllByType(hostType("GestureDetector")) + expect(detectors.length).toBeGreaterThan(0) + for (const detector of detectors) { + expect(hasAncestorOfType(detector, "GestureHandlerRootView")).toBe(true) + } + }) +}) diff --git a/packages/expo/src/wizard/sheet.tsx b/packages/expo/src/wizard/sheet.tsx index 28ca3515..2d99decf 100644 --- a/packages/expo/src/wizard/sheet.tsx +++ b/packages/expo/src/wizard/sheet.tsx @@ -8,6 +8,7 @@ import { Text, View, } from "react-native" +import { GestureHandlerRootView } from "react-native-gesture-handler" import { FlattenView, type FlattenHandle } from "../capture/flatten" import { StepForm } from "./step-form" import { StepAnnotate } from "./step-annotate" @@ -176,132 +177,140 @@ export function WizardSheet({ return ( - - - {/* Header */} - - - - - Repro - - - Report a bug - - - ({ - width: 36, - height: 36, - borderRadius: 18, + {/* On Android a Modal is a separate native window (a Dialog) that sits + outside the host app's GestureHandlerRootView, so gesture-handler + receives no touches inside it and the annotation canvas cannot be + drawn on. Gesture-handler detects this case and activates a nested + root, so the wizard has to carry its own. iOS attaches recognizers to + the views directly and never needed this. */} + + + + {/* Header */} + + - - + + + Repro + + + Report a bug + + + ({ + width: 36, + height: 36, + borderRadius: 18, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.color.surface, + opacity: pressed ? 0.6 : 1, + })} + accessibilityLabel="Close" + accessibilityRole="button" + > + + + + - - - + - {/* Body */} - {step === "form" && ( - - )} - {step === "annotate" && ( - - )} - {step === "submit" && } + {/* Body */} + {step === "form" && ( + + )} + {step === "annotate" && ( + + )} + {step === "submit" && } - {/* Offscreen flatten — always mounted when we have a screenshot and + {/* Offscreen flatten — always mounted when we have a screenshot and measurement so flatten() can run from any step. */} - {screenshot && annotateSize.w > 0 && annotateSize.h > 0 && ( - - )} - - {/* Footer */} - - {step !== "form" ? ( - - ) : null} - - 0 && annotateSize.h > 0 && ( + + )} + + {/* Footer */} + + {step !== "form" ? ( + + ) : null} + + + - - - - setSourcePickerVisible(false)} - /> + + + setSourcePickerVisible(false)} + /> + ) } diff --git a/scripts/test-sdk.sh b/scripts/test-sdk.sh index 57016fa8..68ca42be 100755 --- a/scripts/test-sdk.sh +++ b/scripts/test-sdk.sh @@ -36,5 +36,8 @@ for pkg in "${PACKAGES[@]}"; do while IFS= read -r f; do echo "--- packages/$pkg/$f" (cd "packages/$pkg" && bun test "$f") - done < <(cd "packages/$pkg" && find . -type f -name '*.test.ts' | sed 's|^\./||' | sort) + # Both extensions: the Expo package's component tests are .test.tsx, and a + # find that matched only *.test.ts would skip them silently — CI would go + # green without ever running them. + done < <(cd "packages/$pkg" && find . -type f \( -name '*.test.ts' -o -name '*.test.tsx' \) | sed 's|^\./||' | sort) done From f9f857237a773521778b9ef6cda8b09a07dadd6b Mon Sep 17 00:00:00 2001 From: JJ Date: Mon, 21 Sep 2026 18:03:56 +0800 Subject: [PATCH 2/3] feat(expo): select and drag placed annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Annotations were write-once: a text label that landed in the wrong spot could only be undone and retyped, and the same went for arrows and boxes. The toolbar gains a cursor tool — tap a shape to select it, drag to move it, tap empty canvas to deselect. The geometry is pure and lives in @reprojs/sdk-utils (shapeBounds, hitTest, translateShape, clampTranslation) so the web widget can adopt the same behaviour later; nothing in packages/ui changes here. Shapes are grabbed by their ink — arrows and pen strokes by distance to the line, boxes and text anywhere inside — because a 2px border is not a finger-sized target. Text needs an estimated box: react-native offers no synchronous measurement, and SVG text is positioned by its baseline. "select" stays out of sdk-utils' Tool union, which means "a tool that produces a shape" and backs an exhaustive Record in the web canvas; the expo canvas takes a local CanvasMode instead. The store moves from append-only to snapshot history, because undo now has to reverse edits as well as additions — popping the last shape would delete the label you just nudged instead of putting it back. A whole drag commits once, on release, so it costs one history entry rather than one per frame. Drags are clamped to the canvas: a shape dragged past the edge would leave nothing to tap to get it back. Co-Authored-By: Claude Opus 5 (1M context) --- docs/guide/expo.md | 17 +- packages/expo/README.md | 7 + packages/expo/src/annotation/canvas.test.tsx | 184 +++++++++++++++ packages/expo/src/annotation/canvas.tsx | 214 ++++++++++++----- packages/expo/src/annotation/icons.tsx | 14 ++ packages/expo/src/annotation/mode.ts | 14 ++ packages/expo/src/annotation/store.test.ts | 76 ++++++ packages/expo/src/annotation/store.ts | 60 +++-- packages/expo/src/annotation/toolbar.tsx | 46 ++-- packages/expo/src/wizard/step-annotate.tsx | 20 +- .../sdk-utils/src/annotation/hit-test.test.ts | 219 ++++++++++++++++++ packages/sdk-utils/src/annotation/hit-test.ts | 177 ++++++++++++++ packages/sdk-utils/src/annotation/index.ts | 1 + 13 files changed, 949 insertions(+), 100 deletions(-) create mode 100644 packages/expo/src/annotation/canvas.test.tsx create mode 100644 packages/expo/src/annotation/mode.ts create mode 100644 packages/sdk-utils/src/annotation/hit-test.test.ts create mode 100644 packages/sdk-utils/src/annotation/hit-test.ts diff --git a/docs/guide/expo.md b/docs/guide/expo.md index 59746067..9b93d003 100644 --- a/docs/guide/expo.md +++ b/docs/guide/expo.md @@ -149,11 +149,26 @@ Marking the key `sensitive` hides it from EAS build logs, but `EXPO_PUBLIC_*` is 2. Tap the flame-orange bug button. 3. Fill out title + description → **Continue**. -4. Annotate the captured screenshot with the pen/arrow/rect/highlight/text tools → **Continue**. +4. Annotate the captured screenshot with the pen/arrow/rect/highlight/text tools, or pick the cursor tool to reposition something you already placed → **Continue**. 5. Review what's included → **Send report**. The report lands in your dashboard's inbox with a Mobile / iOS / Android platform pill and a mobile-specific device card. +## Moving an annotation + +The last button in the annotation toolbar is the cursor tool. With it active: + +- Tap a shape to select it — a dashed outline appears around it. +- Drag the shape to move it. Text labels, arrows, rectangles, highlights and pen strokes all move. +- Tap empty canvas to deselect. + +A move is a single undo step, so **Undo** puts the shape back where it was instead of deleting it. +Shapes cannot be dragged off the canvas, and the selection outline is never part of the submitted +PNG. Shapes are grabbed by their ink: an arrow or pen stroke responds near the line itself, while +rectangles, highlights and text can be grabbed anywhere inside them. + +Resizing and rotating are not supported — redraw the shape instead. + ## Draggable launcher The launcher drags AssistiveTouch-style: free movement during the drag, and on release it springs to the nearest screen edge (left, right, top, or bottom) with the position along that edge preserved. It can never end up parked in the middle of the screen. The chosen edge + along-axis position persists across app restarts via AsyncStorage. diff --git a/packages/expo/README.md b/packages/expo/README.md index c17f59be..f88c5f02 100644 --- a/packages/expo/README.md +++ b/packages/expo/README.md @@ -91,6 +91,13 @@ This SDK depends on `react-native-view-shot`, which requires a development build Those are native builds: iOS needs Xcode **plus its iOS platform component** (Xcode → Settings → Components, or `xcodebuild -downloadPlatform iOS`) and a simulator runtime; Android needs an AVD created in Android Studio's Device Manager. Without them `run:ios` / `run:android` fails to find any destination. +## Annotating + +The annotate step offers pen, arrow, rectangle, highlight and text tools, plus a cursor tool for +moving what you already placed: tap a shape to select it, drag to move it, tap empty canvas to +deselect. A move is one undo step, and the selection outline never reaches the submitted PNG. +Resizing and rotating are not supported. + ## What gets captured Every report bundles: diff --git a/packages/expo/src/annotation/canvas.test.tsx b/packages/expo/src/annotation/canvas.test.tsx new file mode 100644 index 00000000..dd3ae438 --- /dev/null +++ b/packages/expo/src/annotation/canvas.test.tsx @@ -0,0 +1,184 @@ +import { describe, expect, test } from "bun:test" +import React from "react" +import { act, create, type ReactTestInstance } from "react-test-renderer" +import { + fireGesture, + gestureOfKind, + hostType, + installNativeMocks, +} from "../test-support/native-mocks" +import type { CanvasMode } from "./mode" + +installNativeMocks() + +const CANVAS = { width: 300, height: 500 } + +function rect(id: string, x: number, y: number) { + return { kind: "rect", id, color: "#e53935", strokeWidth: 4, x, y, w: 60, h: 40 } +} + +async function mountCanvas(mode: CanvasMode, shapes: unknown[] = []) { + const { AnnotationCanvas } = await import("./canvas") + const { createAnnotationStore } = await import("./store") + const store = createAnnotationStore() + for (const shape of shapes) { + // The harness feeds in fully-formed shapes; the store only stores them. + store.addShape(shape as Parameters[0]) + } + + const selections: Array = [] + const textTaps: Array<{ x: number; y: number }> = [] + + // selectedId has to be real state, or the canvas would never re-render with + // a selection and the outline assertion would pass vacuously. + function Harness() { + const [selectedId, setSelectedId] = React.useState(null) + return ( + { + selections.push(id) + setSelectedId(id) + }} + onTextTap={(point) => textTaps.push(point)} + /> + ) + } + + let tree: ReturnType | null = null + act(() => { + tree = create() + }) + if (tree === null) throw new Error("canvas did not mount") + const mounted: ReturnType = tree + return { + detector: mounted.root.findByType(hostType("GestureDetector")), + /** Last id handed to onSelectedIdChange, or undefined if never called. */ + selection: () => selections[selections.length - 1], + textTaps, + store, + rectCount: () => mounted.root.findAllByType(hostType("Rect")).length, + } +} + +function drag( + detector: ReactTestInstance, + from: { x: number; y: number }, + by: { x: number; y: number }, +) { + const pan = gestureOfKind(detector.props.gesture, "pan") + const at = (dx: number, dy: number) => ({ + x: from.x + dx, + y: from.y + dy, + translationX: dx, + translationY: dy, + }) + act(() => { + fireGesture(pan, "onStart", at(0, 0)) + }) + act(() => { + fireGesture(pan, "onUpdate", at(by.x, by.y)) + }) + act(() => { + fireGesture(pan, "onEnd", at(by.x, by.y)) + }) +} + +function tap(detector: ReactTestInstance, at: { x: number; y: number }) { + const gesture = gestureOfKind(detector.props.gesture, "tap") + act(() => { + fireGesture(gesture, "onEnd", at) + }) +} + +describe("AnnotationCanvas select mode", () => { + test("tapping a shape selects it", async () => { + const { detector, selection } = await mountCanvas("select", [rect("r1", 20, 20)]) + tap(detector, { x: 40, y: 30 }) + expect(selection()).toBe("r1") + }) + + test("tapping empty canvas clears the selection", async () => { + const { detector, selection } = await mountCanvas("select", [rect("r1", 20, 20)]) + tap(detector, { x: 40, y: 30 }) + tap(detector, { x: 280, y: 480 }) + expect(selection()).toBeNull() + }) + + test("dragging a shape commits the move to the store", async () => { + const { detector, store } = await mountCanvas("select", [rect("r1", 20, 20)]) + drag(detector, { x: 40, y: 30 }, { x: 50, y: 60 }) + expect(store.snapshot()[0]).toMatchObject({ id: "r1", x: 70, y: 80 }) + }) + + test("a drag is one undo step, and undo restores the original position", async () => { + const { detector, store } = await mountCanvas("select", [rect("r1", 20, 20)]) + drag(detector, { x: 40, y: 30 }, { x: 50, y: 60 }) + act(() => { + store.undo() + }) + expect(store.snapshot()[0]).toMatchObject({ id: "r1", x: 20, y: 20 }) + }) + + test("a drag past the edge is clamped so the shape stays reachable", async () => { + const { detector, store } = await mountCanvas("select", [rect("r1", 20, 20)]) + drag(detector, { x: 40, y: 30 }, { x: -999, y: -999 }) + expect(store.snapshot()[0]).toMatchObject({ id: "r1", x: 0, y: 0 }) + }) + + test("dragging from empty space moves nothing", async () => { + const { detector, store } = await mountCanvas("select", [rect("r1", 20, 20)]) + drag(detector, { x: 250, y: 450 }, { x: -30, y: -30 }) + expect(store.snapshot()[0]).toMatchObject({ id: "r1", x: 20, y: 20 }) + // No history step was recorded, so the only undo available is still the + // one that removes the seeded shape. + act(() => { + store.undo() + }) + expect(store.snapshot()).toHaveLength(0) + }) + + test("select mode never adds a shape", async () => { + const { detector, store } = await mountCanvas("select") + drag(detector, { x: 10, y: 10 }, { x: 80, y: 80 }) + expect(store.snapshot()).toHaveLength(0) + }) + + test("the selected shape gains a selection outline", async () => { + // The outline is drawn by the canvas only. FlattenView renders shapes + // through renderShape, which knows nothing about selection, so a selected + // shape can never bake its outline into the submitted screenshot. + const { detector, rectCount } = await mountCanvas("select", [rect("r1", 20, 20)]) + const before = rectCount() + tap(detector, { x: 40, y: 30 }) + expect(rectCount()).toBe(before + 1) + }) +}) + +describe("AnnotationCanvas drawing modes", () => { + test("pen mode still draws a stroke", async () => { + const { detector, store } = await mountCanvas("pen") + drag(detector, { x: 10, y: 10 }, { x: 40, y: 40 }) + expect(store.snapshot()).toHaveLength(1) + expect(store.snapshot()[0]?.kind).toBe("pen") + }) + + test("rect mode still draws a rectangle from the drag extent", async () => { + const { detector, store } = await mountCanvas("rect") + drag(detector, { x: 10, y: 20 }, { x: 60, y: 40 }) + expect(store.snapshot()[0]).toMatchObject({ kind: "rect", x: 10, y: 20, w: 60, h: 40 }) + }) + + test("text mode reports the tap position instead of drawing", async () => { + const { detector, store, textTaps } = await mountCanvas("text") + tap(detector, { x: 33, y: 44 }) + expect(textTaps).toEqual([{ x: 33, y: 44 }]) + expect(store.snapshot()).toHaveLength(0) + }) +}) diff --git a/packages/expo/src/annotation/canvas.tsx b/packages/expo/src/annotation/canvas.tsx index 6cc2894e..3ea04501 100644 --- a/packages/expo/src/annotation/canvas.tsx +++ b/packages/expo/src/annotation/canvas.tsx @@ -1,34 +1,106 @@ -import React, { useState } from "react" +import React, { useRef, useState } from "react" import { View } from "react-native" import { GestureDetector, Gesture } from "react-native-gesture-handler" -import Svg from "react-native-svg" +import Svg, { Rect } from "react-native-svg" import type { AnnotationStore } from "./store" -import type { Shape, Tool, PenPoint } from "@reprojs/sdk-utils" -import { newShapeId } from "@reprojs/sdk-utils" +import type { Shape, PenPoint } from "@reprojs/sdk-utils" +import { + clampTranslation, + hitTest, + newShapeId, + shapeBounds, + translateShape, +} from "@reprojs/sdk-utils" +import type { CanvasMode } from "./mode" import { useAnnotationShapes } from "./use-shapes" import { renderShape } from "./render-shape" interface Props { width: number height: number - tool: Tool + mode: CanvasMode color: string strokeWidth: number store: AnnotationStore + selectedId: string | null + onSelectedIdChange: (id: string | null) => void onTextTap?: (point: { x: number; y: number }) => void } +const DRAFT_ID = "__draft__" + +/** Selection chrome. Deliberately near-black so it reads as UI rather than + * ink — every drawing colour in the palette is a saturated hue. */ +const SELECTION_COLOR = "#111827" +const SELECTION_PAD = 6 + export function AnnotationCanvas({ width, height, - tool, + mode, color, strokeWidth, store, + selectedId, + onSelectedIdChange, onTextTap, }: Props) { const shapes = useAnnotationShapes(store) const [draftPoints, setDraftPoints] = useState([]) + const [dragPreview, setDragPreview] = useState<{ id: string; dx: number; dy: number } | null>( + null, + ) + + // In-flight gesture state is mirrored into refs. The gesture callbacks below + // are captured when the gesture is built, so reading React state inside them + // can see a value from before the stroke started; a ref is always current. + // State still exists because only state re-renders the preview. + const draftRef = useRef([]) + const dragIdRef = useRef(null) + + function pushDraftPoint(point: PenPoint) { + draftRef.current = [...draftRef.current, point] + setDraftPoints(draftRef.current) + } + + function resetDraft() { + draftRef.current = [] + setDraftPoints([]) + } + + function shapeById(id: string): Shape | null { + return store.snapshot().find((s) => s.id === id) ?? null + } + + function beginDrag(x: number, y: number) { + const hit = hitTest(store.snapshot(), { x, y }) + dragIdRef.current = hit?.id ?? null + setDragPreview(hit === null ? null : { id: hit.id, dx: 0, dy: 0 }) + if (hit !== null) onSelectedIdChange(hit.id) + } + + function updateDrag(translationX: number, translationY: number) { + const id = dragIdRef.current + if (id === null) return + const shape = shapeById(id) + if (shape === null) return + const offset = clampTranslation(shape, translationX, translationY, { w: width, h: height }) + setDragPreview({ id, ...offset }) + } + + function endDrag(translationX: number, translationY: number) { + const id = dragIdRef.current + dragIdRef.current = null + setDragPreview(null) + if (id === null) return + const shape = shapeById(id) + if (shape === null) return + // Committed from the gesture's total translation rather than the preview + // offset, so the stored position is exactly what the reporter released at + // and the whole drag lands as a single undo step. + const offset = clampTranslation(shape, translationX, translationY, { w: width, h: height }) + store.moveShape(id, offset.dx, offset.dy) + } // runOnJS(true) forces callbacks to the JS thread. Without it, callbacks run // as worklets on the UI thread when react-native-reanimated is present, and @@ -37,97 +109,114 @@ export function AnnotationCanvas({ .runOnJS(true) .minDistance(2) .onStart((e) => { - if (tool === "text") return - setDraftPoints([{ x: e.x, y: e.y, p: 1 }]) + if (mode === "select") { + beginDrag(e.x, e.y) + return + } + if (mode === "text") return + resetDraft() + pushDraftPoint({ x: e.x, y: e.y, p: 1 }) }) .onUpdate((e) => { - if (tool === "text") return - setDraftPoints((d) => [...d, { x: e.x, y: e.y, p: 1 }]) + if (mode === "select") { + updateDrag(e.translationX, e.translationY) + return + } + if (mode === "text") return + pushDraftPoint({ x: e.x, y: e.y, p: 1 }) }) - .onEnd(() => { - if (tool === "text") return - if (draftPoints.length === 0) { - setDraftPoints([]) + .onEnd((e) => { + if (mode === "select") { + endDrag(e.translationX, e.translationY) return } - const shape = buildShape(tool, draftPoints, color, strokeWidth) + if (mode === "text") return + const points = draftRef.current + resetDraft() + if (points.length === 0) return + const shape = buildShape(mode, points, color, strokeWidth, newShapeId()) if (shape) store.addShape(shape) - setDraftPoints([]) }) const tap = Gesture.Tap() .runOnJS(true) .onEnd((e) => { - if (tool !== "text") return + if (mode === "select") { + onSelectedIdChange(hitTest(store.snapshot(), { x: e.x, y: e.y })?.id ?? null) + return + } + if (mode !== "text") return onTextTap?.({ x: e.x, y: e.y }) }) const gesture = Gesture.Race(pan, tap) - const draftShape = buildDraftShape(tool, draftPoints, color, strokeWidth) + // The shape under the finger is previewed at its new position without being + // written to the store, so a drag costs one history entry instead of one per + // frame. + const painted = + dragPreview === null + ? shapes + : shapes.map((s) => + s.id === dragPreview.id ? translateShape(s, dragPreview.dx, dragPreview.dy) : s, + ) + + const selected = + mode === "select" && selectedId !== null + ? (painted.find((s) => s.id === selectedId) ?? null) + : null + + const draftShape = buildShape(mode, draftPoints, color, strokeWidth, DRAFT_ID) return ( - {shapes.map((s, i) => renderShape(s, i))} + {painted.map((s, i) => renderShape(s, i))} {draftShape !== null ? renderShape(draftShape, "draft") : null} + {/* Selection chrome lives here and nowhere else. FlattenView renders + shapes through renderShape, which knows nothing about selection, + so the outline can never bake into the submitted PNG. */} + {selected !== null ? : null} ) } -function buildShape( - tool: Tool, - points: PenPoint[], - color: string, - strokeWidth: number, -): Shape | null { - const first = points[0] - const last = points[points.length - 1] - if (!first || !last) return null - const id = newShapeId() - if (tool === "pen") return { kind: "pen", id, color, strokeWidth, points } - if (tool === "arrow") - return { - kind: "arrow", - id, - color, - strokeWidth, - x1: first.x, - y1: first.y, - x2: last.x, - y2: last.y, - } - if (tool === "rect" || tool === "highlight") { - return { - kind: tool, - id, - color, - strokeWidth, - x: Math.min(first.x, last.x), - y: Math.min(first.y, last.y), - w: Math.abs(last.x - first.x), - h: Math.abs(last.y - first.y), - } - } - return null +function SelectionOutline({ shape }: { shape: Shape }) { + const b = shapeBounds(shape) + return ( + + ) } -function buildDraftShape( - tool: Tool, +/** + * Turns a drag into a shape. Returns null for modes that don't draw from a + * drag (`text` is placed by a tap, `select` moves what's already there) and + * for a drag with no points yet. + */ +function buildShape( + mode: CanvasMode, points: PenPoint[], color: string, strokeWidth: number, + id: string, ): Shape | null { - if (tool === "text") return null const first = points[0] const last = points[points.length - 1] if (!first || !last) return null - const id = "__draft__" - if (tool === "pen") return { kind: "pen", id, color, strokeWidth, points } - if (tool === "arrow") + if (mode === "pen") return { kind: "pen", id, color, strokeWidth, points } + if (mode === "arrow") { return { kind: "arrow", id, @@ -138,9 +227,10 @@ function buildDraftShape( x2: last.x, y2: last.y, } - if (tool === "rect" || tool === "highlight") { + } + if (mode === "rect" || mode === "highlight") { return { - kind: tool, + kind: mode, id, color, strokeWidth, diff --git a/packages/expo/src/annotation/icons.tsx b/packages/expo/src/annotation/icons.tsx index d875b2fe..3b1b153e 100644 --- a/packages/expo/src/annotation/icons.tsx +++ b/packages/expo/src/annotation/icons.tsx @@ -137,3 +137,17 @@ export function TrashIcon({ size = 20, color = "#111827" }: IconProps) { ) } + +export function CursorIcon({ size = 20, color = "#111827" }: IconProps) { + return ( + + + + ) +} diff --git a/packages/expo/src/annotation/mode.ts b/packages/expo/src/annotation/mode.ts new file mode 100644 index 00000000..2b67b193 --- /dev/null +++ b/packages/expo/src/annotation/mode.ts @@ -0,0 +1,14 @@ +import type { Tool } from "@reprojs/sdk-utils" + +/** + * What the canvas does with a gesture: draw with one of the shared drawing + * tools, or select and move what is already drawn. + * + * `select` is deliberately *not* a member of `@reprojs/sdk-utils`'s `Tool` + * union. `Tool` means "a tool that produces a shape", and the web widget keys + * an exhaustive `Record` off it — adding a member there + * would break that build for a mode the web widget does not have. + */ +export type CanvasMode = Tool | "select" + +export const SELECT_MODE: CanvasMode = "select" diff --git a/packages/expo/src/annotation/store.test.ts b/packages/expo/src/annotation/store.test.ts index 59c5f4fb..19d8b3eb 100644 --- a/packages/expo/src/annotation/store.test.ts +++ b/packages/expo/src/annotation/store.test.ts @@ -69,3 +69,79 @@ test("canRedo is false after addShape clears redo stack", () => { s.addShape(rect("#0f0", 1)) expect(s.canRedo()).toBe(false) }) + +test("moveShape offsets the matching shape and leaves others alone", () => { + const s = createAnnotationStore() + const a = rect("#f00", 0) + const b = rect("#0f0", 100) + s.addShape(a) + s.addShape(b) + s.moveShape(a.id, 5, 7) + const [movedA, untouchedB] = s.snapshot() + expect(movedA).toMatchObject({ id: a.id, x: 5, y: 7 }) + expect(untouchedB).toMatchObject({ id: b.id, x: 100, y: 0 }) +}) + +test("moveShape keeps the shape's place in the paint order", () => { + const s = createAnnotationStore() + const first = rect("#f00", 0) + const second = rect("#0f0", 100) + s.addShape(first) + s.addShape(second) + s.moveShape(first.id, 10, 0) + expect(s.snapshot().map((x) => x.id)).toEqual([first.id, second.id]) +}) + +test("undo after a move restores the old position instead of deleting the shape", () => { + const s = createAnnotationStore() + const r = rect("#f00", 0) + s.addShape(r) + s.moveShape(r.id, 40, 40) + s.undo() + expect(s.snapshot()).toHaveLength(1) + expect(s.snapshot()[0]).toMatchObject({ x: 0, y: 0 }) + s.redo() + expect(s.snapshot()[0]).toMatchObject({ x: 40, y: 40 }) +}) + +test("moveShape with an unknown id records no history step", () => { + const s = createAnnotationStore() + s.addShape(rect()) + s.undo() + expect(s.canUndo()).toBe(false) + s.moveShape("nope", 10, 10) + expect(s.canUndo()).toBe(false) + expect(s.canRedo()).toBe(true) +}) + +test("a zero-distance move records no history step", () => { + const s = createAnnotationStore() + const r = rect() + s.addShape(r) + s.moveShape(r.id, 0, 0) + s.undo() + expect(s.snapshot()).toHaveLength(0) +}) + +test("moveShape discards the redo stack", () => { + const s = createAnnotationStore() + const r = rect("#f00", 0) + s.addShape(r) + s.addShape(rect("#0f0", 50)) + s.undo() + expect(s.canRedo()).toBe(true) + s.moveShape(r.id, 1, 1) + expect(s.canRedo()).toBe(false) +}) + +test("moveShape notifies subscribers", () => { + const s = createAnnotationStore() + const r = rect() + s.addShape(r) + let calls = 0 + s.subscribe(() => { + calls++ + }) + s.moveShape(r.id, 2, 2) + expect(calls).toBe(1) +}) diff --git a/packages/expo/src/annotation/store.ts b/packages/expo/src/annotation/store.ts index d02078b2..2431bc08 100644 --- a/packages/expo/src/annotation/store.ts +++ b/packages/expo/src/annotation/store.ts @@ -1,7 +1,10 @@ import type { Shape } from "@reprojs/sdk-utils" +import { translateShape } from "@reprojs/sdk-utils" export interface AnnotationStore { addShape: (s: Shape) => void + /** Offsets one shape by (dx, dy) as a single undoable step. */ + moveShape: (id: string, dx: number, dy: number) => void undo: () => void redo: () => void clear: () => void @@ -11,46 +14,67 @@ export interface AnnotationStore { canRedo: () => boolean } +/** + * History is kept as whole-list snapshots rather than a stack of added shapes, + * because undo has to reverse edits as well as additions: once a shape can be + * dragged, "pop the last shape" would delete the label you just nudged instead + * of putting it back where it was. Annotations number in the tens, so copying + * the list per edit costs nothing. + */ export function createAnnotationStore(): AnnotationStore { - let shapes: Shape[] = [] - let redoStack: Shape[] = [] + let past: Shape[][] = [] + let present: Shape[] = [] + let future: Shape[][] = [] const listeners = new Set<() => void>() const notify = () => { for (const l of listeners) l() } + /** Commits a new shape list, making the previous one undoable. */ + const commit = (next: Shape[]) => { + past = [...past, present] + present = next + future = [] + notify() + } return { addShape(s) { - shapes = [...shapes, s] - redoStack = [] - notify() + commit([...present, s]) + }, + moveShape(id, dx, dy) { + if (dx === 0 && dy === 0) return + if (!present.some((s) => s.id === id)) return + commit(present.map((s) => (s.id === id ? translateShape(s, dx, dy) : s))) }, undo() { - if (!shapes.length) return - const popped = shapes[shapes.length - 1] - shapes = shapes.slice(0, -1) - if (popped) redoStack = [...redoStack, popped] + const previous = past[past.length - 1] + if (previous === undefined) return + past = past.slice(0, -1) + future = [present, ...future] + present = previous notify() }, redo() { - if (!redoStack.length) return - const last = redoStack[redoStack.length - 1] - redoStack = redoStack.slice(0, -1) - if (last) shapes = [...shapes, last] + const next = future[0] + if (next === undefined) return + future = future.slice(1) + past = [...past, present] + present = next notify() }, clear() { - shapes = [] - redoStack = [] + past = [] + present = [] + future = [] notify() }, - snapshot: () => shapes, + snapshot: () => present, subscribe(fn) { listeners.add(fn) return () => { listeners.delete(fn) } }, - canUndo: () => shapes.length > 0, - canRedo: () => redoStack.length > 0, + canUndo: () => past.length > 0, + canRedo: () => future.length > 0, } } diff --git a/packages/expo/src/annotation/toolbar.tsx b/packages/expo/src/annotation/toolbar.tsx index a85adb54..02e9bacd 100644 --- a/packages/expo/src/annotation/toolbar.tsx +++ b/packages/expo/src/annotation/toolbar.tsx @@ -1,9 +1,10 @@ import React from "react" import { Pressable, View } from "react-native" -import type { Tool } from "@reprojs/sdk-utils" import { PALETTE, STROKE_WIDTHS } from "@reprojs/sdk-utils" +import type { CanvasMode } from "./mode" import type { AnnotationStore } from "./store" import { + CursorIcon, PenIcon, ArrowIcon, RectIcon, @@ -16,8 +17,8 @@ import { import { useAnnotationShapes } from "./use-shapes" interface Props { - tool: Tool - onToolChange: (t: Tool) => void + mode: CanvasMode + onModeChange: (m: CanvasMode) => void color: string onColorChange: (c: string) => void strokeWidth: number @@ -25,12 +26,21 @@ interface Props { store: AnnotationStore } -const TOOLS: { key: Tool; Icon: React.ComponentType<{ size?: number; color?: string }> }[] = [ - { key: "pen", Icon: PenIcon }, - { key: "arrow", Icon: ArrowIcon }, - { key: "rect", Icon: RectIcon }, - { key: "highlight", Icon: HighlightIcon }, - { key: "text", Icon: TextIcon }, +interface ToolButton { + key: CanvasMode + label: string + Icon: React.ComponentType<{ size?: number; color?: string }> +} + +// `select` sits last so the drawing tools keep the positions reporters already +// know. +const TOOLS: ToolButton[] = [ + { key: "pen", label: "Pen", Icon: PenIcon }, + { key: "arrow", label: "Arrow", Icon: ArrowIcon }, + { key: "rect", label: "Rectangle", Icon: RectIcon }, + { key: "highlight", label: "Highlight", Icon: HighlightIcon }, + { key: "text", label: "Text", Icon: TextIcon }, + { key: "select", label: "Select and move", Icon: CursorIcon }, ] const ACTIVE_BG = "#ff9b51" @@ -41,8 +51,8 @@ const DISABLED_OPACITY = 0.35 const HIT_SIZE = 44 export function AnnotationToolbar({ - tool, - onToolChange, + mode, + onModeChange, color, onColorChange, strokeWidth, @@ -59,12 +69,14 @@ export function AnnotationToolbar({ {/* Row 1: tool buttons + undo/redo/trash */} - {TOOLS.map(({ key, Icon }) => { - const active = key === tool + {TOOLS.map(({ key, label, Icon }) => { + const active = key === mode return ( onToolChange(key)} + onPress={() => onModeChange(key)} + accessibilityLabel={label} + accessibilityRole="button" style={{ width: HIT_SIZE, height: HIT_SIZE, @@ -86,6 +98,8 @@ export function AnnotationToolbar({ store.undo()} disabled={!canUndo} + accessibilityLabel="Undo" + accessibilityRole="button" style={{ width: HIT_SIZE, height: HIT_SIZE, @@ -101,6 +115,8 @@ export function AnnotationToolbar({ store.redo()} disabled={!canRedo} + accessibilityLabel="Redo" + accessibilityRole="button" style={{ width: HIT_SIZE, height: HIT_SIZE, @@ -115,6 +131,8 @@ export function AnnotationToolbar({ {/* Trash / clear */} store.clear()} + accessibilityLabel="Clear all annotations" + accessibilityRole="button" style={{ width: HIT_SIZE, height: HIT_SIZE, diff --git a/packages/expo/src/wizard/step-annotate.tsx b/packages/expo/src/wizard/step-annotate.tsx index 9fff5a8c..4406fc97 100644 --- a/packages/expo/src/wizard/step-annotate.tsx +++ b/packages/expo/src/wizard/step-annotate.tsx @@ -4,7 +4,7 @@ import { AnnotationCanvas } from "../annotation/canvas" import { AnnotationToolbar } from "../annotation/toolbar" import { TextInputModal } from "../annotation/text-input-modal" import type { AnnotationStore } from "../annotation/store" -import type { Tool } from "@reprojs/sdk-utils" +import type { CanvasMode } from "../annotation/mode" import { PALETTE, STROKE_WIDTHS, newShapeId } from "@reprojs/sdk-utils" interface Props { @@ -14,12 +14,20 @@ interface Props { } export function StepAnnotate({ imageUri, store, onSizeChange }: Props) { - const [tool, setTool] = useState("pen") + const [mode, setMode] = useState("pen") + const [selectedId, setSelectedId] = useState(null) const [color, setColor] = useState(PALETTE[0]) const [strokeWidth, setStrokeWidth] = useState(STROKE_WIDTHS[1]) const [textPoint, setTextPoint] = useState<{ x: number; y: number } | null>(null) const [size, setSize] = useState({ w: 0, h: 0 }) + function handleModeChange(next: CanvasMode) { + setMode(next) + // A selection outline would be misleading once another tool is active: + // nothing would drag it. + if (next !== "select") setSelectedId(null) + } + function handleTextTap(point: { x: number; y: number }) { setTextPoint(point) } @@ -48,8 +56,8 @@ export function StepAnnotate({ imageUri, store, onSizeChange }: Props) { return ( )} diff --git a/packages/sdk-utils/src/annotation/hit-test.test.ts b/packages/sdk-utils/src/annotation/hit-test.test.ts new file mode 100644 index 00000000..0708a4ce --- /dev/null +++ b/packages/sdk-utils/src/annotation/hit-test.test.ts @@ -0,0 +1,219 @@ +import { expect, test } from "bun:test" +import type { Shape } from "./types" +import { clampTranslation, hitTest, shapeBounds, translateShape } from "./hit-test" + +function rect(over: Partial> = {}): Shape { + return { + kind: "rect", + id: "r1", + color: "#f00", + strokeWidth: 2, + x: 10, + y: 20, + w: 40, + h: 30, + ...over, + } +} + +function text(over: Partial> = {}): Shape { + return { + kind: "text", + id: "t1", + color: "#f00", + strokeWidth: 2, + x: 100, + y: 100, + w: 0, + h: 0, + content: "hello", + fontSize: 16, + ...over, + } +} + +function arrow(over: Partial> = {}): Shape { + return { + kind: "arrow", + id: "a1", + color: "#f00", + strokeWidth: 2, + x1: 0, + y1: 0, + x2: 100, + y2: 100, + ...over, + } +} + +function pen(points: Array<{ x: number; y: number }>): Shape { + return { + kind: "pen", + id: "p1", + color: "#f00", + strokeWidth: 2, + points: points.map((p) => ({ ...p, p: 1 })), + } +} + +// --- shapeBounds ------------------------------------------------------------- + +test("shapeBounds returns the rect itself", () => { + expect(shapeBounds(rect())).toEqual({ x: 10, y: 20, w: 40, h: 30 }) +}) + +test("shapeBounds normalises a rect drawn with negative extent", () => { + expect(shapeBounds(rect({ x: 50, y: 50, w: -20, h: -10 }))).toEqual({ + x: 30, + y: 40, + w: 20, + h: 10, + }) +}) + +test("shapeBounds spans both arrow endpoints regardless of direction", () => { + expect(shapeBounds(arrow({ x1: 100, y1: 80, x2: 20, y2: 10 }))).toEqual({ + x: 20, + y: 10, + w: 80, + h: 70, + }) +}) + +test("shapeBounds covers every pen point", () => { + const b = shapeBounds( + pen([ + { x: 5, y: 5 }, + { x: 25, y: 15 }, + { x: 15, y: 40 }, + ]), + ) + expect(b).toEqual({ x: 5, y: 5, w: 20, h: 35 }) +}) + +test("shapeBounds of a text sits above the baseline", () => { + // SVG text is positioned by its baseline, so the box has to extend upwards + // from `y` or a tap on the glyphs would miss it entirely. + const b = shapeBounds(text({ x: 100, y: 100, fontSize: 20 })) + expect(b.x).toBe(100) + expect(b.y).toBeLessThan(100) + expect(b.y + b.h).toBeGreaterThan(100) + expect(b.w).toBeGreaterThan(0) +}) + +test("shapeBounds of a text scales with content length", () => { + const short = shapeBounds(text({ content: "hi" })) + const long = shapeBounds(text({ content: "a much longer label" })) + expect(long.w).toBeGreaterThan(short.w) +}) + +test("shapeBounds gives an empty pen shape a zero box rather than infinities", () => { + expect(shapeBounds(pen([]))).toEqual({ x: 0, y: 0, w: 0, h: 0 }) +}) + +// --- hitTest ----------------------------------------------------------------- + +test("hitTest finds a rect when tapped inside", () => { + const r = rect() + expect(hitTest([r], { x: 30, y: 30 })?.id).toBe("r1") +}) + +test("hitTest misses a rect tapped well outside", () => { + expect(hitTest([rect()], { x: 300, y: 300 })).toBeNull() +}) + +test("hitTest forgives a near miss by the slop margin", () => { + // Fingers are imprecise: just outside the edge still counts. + expect(hitTest([rect()], { x: 8, y: 30 }, 12)?.id).toBe("r1") + expect(hitTest([rect()], { x: 8, y: 30 }, 0)).toBeNull() +}) + +test("hitTest returns the topmost shape when two overlap", () => { + const under = rect({ id: "under" }) + const over = rect({ id: "over" }) + expect(hitTest([under, over], { x: 30, y: 30 })?.id).toBe("over") +}) + +test("hitTest matches an arrow by distance to its shaft, not its bounding box", () => { + const a = arrow({ x1: 0, y1: 0, x2: 100, y2: 100 }) + expect(hitTest([a], { x: 50, y: 50 })?.id).toBe("a1") + // Inside the bounding box but far from the diagonal line. + expect(hitTest([a], { x: 95, y: 5 })).toBeNull() +}) + +test("hitTest matches a pen stroke near any of its segments", () => { + const p = pen([ + { x: 0, y: 0 }, + { x: 100, y: 0 }, + { x: 100, y: 100 }, + ]) + expect(hitTest([p], { x: 50, y: 2 })?.id).toBe("p1") + expect(hitTest([p], { x: 98, y: 80 })?.id).toBe("p1") + // The open corner of the L — inside the box, nowhere near the ink. + expect(hitTest([p], { x: 10, y: 90 })).toBeNull() +}) + +test("hitTest matches a text label by its estimated box", () => { + expect(hitTest([text({ x: 100, y: 100 })], { x: 105, y: 95 })?.id).toBe("t1") +}) + +test("hitTest on an empty list is null", () => { + expect(hitTest([], { x: 0, y: 0 })).toBeNull() +}) + +// --- translateShape ---------------------------------------------------------- + +test("translateShape moves a rect and leaves the original untouched", () => { + const r = rect() + const moved = translateShape(r, 5, -10) + expect(moved).toMatchObject({ x: 15, y: 10, w: 40, h: 30 }) + expect(r).toMatchObject({ x: 10, y: 20 }) +}) + +test("translateShape moves both arrow endpoints", () => { + const moved = translateShape(arrow(), 10, 20) + expect(moved).toMatchObject({ x1: 10, y1: 20, x2: 110, y2: 120 }) +}) + +test("translateShape moves every pen point", () => { + const moved = translateShape( + pen([ + { x: 0, y: 0 }, + { x: 10, y: 10 }, + ]), + 3, + 4, + ) + expect(moved.kind === "pen" && moved.points).toEqual([ + { x: 3, y: 4, p: 1 }, + { x: 13, y: 14, p: 1 }, + ]) +}) + +test("translateShape keeps the shape id and style", () => { + const moved = translateShape(text(), 1, 1) + expect(moved.id).toBe("t1") + expect(moved.color).toBe("#f00") + expect(moved.kind === "text" && moved.content).toBe("hello") +}) + +// --- clampTranslation -------------------------------------------------------- + +test("clampTranslation passes a move that stays inside through unchanged", () => { + expect(clampTranslation(rect(), 5, 5, { w: 200, h: 200 })).toEqual({ dx: 5, dy: 5 }) +}) + +test("clampTranslation stops a shape at the left and top edges", () => { + // rect sits at x:10 y:20, so it can only move -10 / -20 before clipping. + expect(clampTranslation(rect(), -50, -60, { w: 200, h: 200 })).toEqual({ dx: -10, dy: -20 }) +}) + +test("clampTranslation stops a shape at the right and bottom edges", () => { + // rect is 40x30 inside a 100x100 canvas: 50 left horizontally, 50 vertically. + expect(clampTranslation(rect(), 999, 999, { w: 100, h: 100 })).toEqual({ dx: 50, dy: 50 }) +}) + +test("clampTranslation leaves a shape larger than the canvas free to move", () => { + const huge = rect({ x: 0, y: 0, w: 500, h: 500 }) + expect(clampTranslation(huge, -100, -100, { w: 100, h: 100 })).toEqual({ dx: -100, dy: -100 }) +}) diff --git a/packages/sdk-utils/src/annotation/hit-test.ts b/packages/sdk-utils/src/annotation/hit-test.ts new file mode 100644 index 00000000..12229f8a --- /dev/null +++ b/packages/sdk-utils/src/annotation/hit-test.ts @@ -0,0 +1,177 @@ +import type { Shape } from "./types" + +export interface Bounds { + x: number + y: number + w: number + h: number +} + +export interface Point { + x: number + y: number +} + +/** + * Default forgiveness, in canvas units, when deciding whether a tap landed on + * a shape. A fingertip covers far more than a 2px stroke, so every hit test + * inflates the target by at least this much. + */ +export const DEFAULT_HIT_SLOP = 12 + +/** + * Text metrics we have to guess at. React Native gives no synchronous text + * measurement, and the annotation canvas needs a box *now* to hit-test and to + * draw a selection outline, so the box is estimated from the font size: + * + * - `TEXT_ADVANCE_RATIO` — mean glyph advance as a fraction of font size. + * 0.6 is the usual approximation for a proportional sans-serif. + * - `TEXT_ASCENT_RATIO` — how far the glyphs rise above the baseline. SVG + * text is positioned by its baseline, so the box starts above `y`. + * - `TEXT_LINE_RATIO` — full line height, ascent plus descent. + * + * The estimate is deliberately generous: an outline slightly larger than the + * glyphs looks intentional, while one that clips them looks broken. + */ +export const TEXT_ADVANCE_RATIO = 0.6 +export const TEXT_ASCENT_RATIO = 0.8 +export const TEXT_LINE_RATIO = 1.2 + +/** Axis-aligned box enclosing a shape, in canvas coordinates. */ +export function shapeBounds(shape: Shape): Bounds { + if (shape.kind === "rect" || shape.kind === "highlight") { + return normalize(shape.x, shape.y, shape.w, shape.h) + } + if (shape.kind === "arrow") { + return normalize(shape.x1, shape.y1, shape.x2 - shape.x1, shape.y2 - shape.y1) + } + if (shape.kind === "text") { + const w = Math.max(shape.content.length * shape.fontSize * TEXT_ADVANCE_RATIO, shape.fontSize) + return { + x: shape.x, + y: shape.y - shape.fontSize * TEXT_ASCENT_RATIO, + w, + h: shape.fontSize * TEXT_LINE_RATIO, + } + } + if (shape.points.length === 0) return { x: 0, y: 0, w: 0, h: 0 } + let minX = Infinity + let minY = Infinity + let maxX = -Infinity + let maxY = -Infinity + for (const p of shape.points) { + minX = Math.min(minX, p.x) + minY = Math.min(minY, p.y) + maxX = Math.max(maxX, p.x) + maxY = Math.max(maxY, p.y) + } + return { x: minX, y: minY, w: maxX - minX, h: maxY - minY } +} + +/** + * Topmost shape under `point`, or null. Later shapes win, matching what the + * reporter sees: shapes are painted in insertion order, so the last one drawn + * is the one on top. + * + * Filled and box-like shapes (rect, highlight, text) are grabbable anywhere + * inside them — an unfilled rectangle's 2px border is far too thin to catch + * with a finger. Arrows and pen strokes are matched against the ink itself, so + * a long diagonal can't be picked up from the empty corner of its bounding box. + */ +export function hitTest( + shapes: Shape[], + point: Point, + slop: number = DEFAULT_HIT_SLOP, +): Shape | null { + for (let i = shapes.length - 1; i >= 0; i--) { + const shape = shapes[i] + if (shape && isHit(shape, point, slop)) return shape + } + return null +} + +/** Copy of `shape` offset by (dx, dy). The input is never mutated. */ +export function translateShape(shape: Shape, dx: number, dy: number): Shape { + if (shape.kind === "arrow") { + return { ...shape, x1: shape.x1 + dx, y1: shape.y1 + dy, x2: shape.x2 + dx, y2: shape.y2 + dy } + } + if (shape.kind === "pen") { + return { ...shape, points: shape.points.map((p) => ({ ...p, x: p.x + dx, y: p.y + dy })) } + } + return { ...shape, x: shape.x + dx, y: shape.y + dy } +} + +/** + * Trims a drag so the shape stays fully on the canvas. Without this a shape + * dragged past the edge is unreachable forever — there is nothing left to tap + * to select it again, and only undo can bring it back. + * + * A shape bigger than the canvas in one axis is left unconstrained on that + * axis, since no offset could satisfy the rule and clamping would freeze it. + */ +export function clampTranslation( + shape: Shape, + dx: number, + dy: number, + canvas: { w: number; h: number }, +): { dx: number; dy: number } { + const b = shapeBounds(shape) + return { + dx: b.w <= canvas.w ? clamp(dx, -b.x, canvas.w - (b.x + b.w)) : dx, + dy: b.h <= canvas.h ? clamp(dy, -b.y, canvas.h - (b.y + b.h)) : dy, + } +} + +function isHit(shape: Shape, point: Point, slop: number): boolean { + const reach = slop + shape.strokeWidth / 2 + if (shape.kind === "arrow") { + return distanceToSegment(point, shape.x1, shape.y1, shape.x2, shape.y2) <= reach + } + if (shape.kind === "pen") { + const pts = shape.points + if (pts.length === 1) { + const only = pts[0] + return only !== undefined && distance(point, only) <= reach + } + for (let i = 1; i < pts.length; i++) { + const a = pts[i - 1] + const b = pts[i] + if (a && b && distanceToSegment(point, a.x, a.y, b.x, b.y) <= reach) return true + } + return false + } + const b = shapeBounds(shape) + return ( + point.x >= b.x - slop && + point.x <= b.x + b.w + slop && + point.y >= b.y - slop && + point.y <= b.y + b.h + slop + ) +} + +function distanceToSegment(p: Point, x1: number, y1: number, x2: number, y2: number): number { + const dx = x2 - x1 + const dy = y2 - y1 + const lengthSq = dx * dx + dy * dy + if (lengthSq === 0) return distance(p, { x: x1, y: y1 }) + // Projection of p onto the segment, clamped to the segment's ends. + const t = clamp(((p.x - x1) * dx + (p.y - y1) * dy) / lengthSq, 0, 1) + return distance(p, { x: x1 + t * dx, y: y1 + t * dy }) +} + +function distance(a: Point, b: Point): number { + return Math.hypot(a.x - b.x, a.y - b.y) +} + +function normalize(x: number, y: number, w: number, h: number): Bounds { + return { + x: w < 0 ? x + w : x, + y: h < 0 ? y + h : y, + w: Math.abs(w), + h: Math.abs(h), + } +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max) +} diff --git a/packages/sdk-utils/src/annotation/index.ts b/packages/sdk-utils/src/annotation/index.ts index 64108939..e5914fa5 100644 --- a/packages/sdk-utils/src/annotation/index.ts +++ b/packages/sdk-utils/src/annotation/index.ts @@ -6,3 +6,4 @@ export * from "./tools/pen" export * from "./tools/rect" export * from "./tools/highlight" export * from "./tools/text" +export * from "./hit-test" From caa706b425444d5b0050e59aabc80e53dedfc618 Mon Sep 17 00:00:00 2001 From: JJ Date: Mon, 21 Sep 2026 18:05:44 +0800 Subject: [PATCH 3/3] fix(expo): let the annotation toolbar shrink to fit narrow phones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cursor tool makes nine buttons in the first toolbar row. At 44pt each plus gaps that is ~416pt, which does not fit a 375pt phone, and Yoga defaults flexShrink to 0 — the row would have run off the edge and taken the trash button with it. The buttons now shrink to no less than 34pt, so they stay at 44pt wherever there is room. Co-Authored-By: Claude Opus 5 (1M context) --- packages/expo/src/annotation/toolbar.tsx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/expo/src/annotation/toolbar.tsx b/packages/expo/src/annotation/toolbar.tsx index 02e9bacd..7a18824a 100644 --- a/packages/expo/src/annotation/toolbar.tsx +++ b/packages/expo/src/annotation/toolbar.tsx @@ -49,6 +49,12 @@ const ACTIVE_ICON = "#ffffff" const INACTIVE_ICON = "#111827" const DISABLED_OPACITY = 0.35 const HIT_SIZE = 44 +/** + * Nine 44pt buttons plus gaps need ~416pt, which overflows a 375pt phone. + * Yoga defaults flexShrink to 0, so without this the row would not shrink — + * it would run off the edge and take the trash button with it. + */ +const MIN_HIT_SIZE = 34 export function AnnotationToolbar({ mode, @@ -79,6 +85,8 @@ export function AnnotationToolbar({ accessibilityRole="button" style={{ width: HIT_SIZE, + minWidth: MIN_HIT_SIZE, + flexShrink: 1, height: HIT_SIZE, borderRadius: 8, backgroundColor: active ? ACTIVE_BG : INACTIVE_BG, @@ -102,6 +110,8 @@ export function AnnotationToolbar({ accessibilityRole="button" style={{ width: HIT_SIZE, + minWidth: MIN_HIT_SIZE, + flexShrink: 1, height: HIT_SIZE, alignItems: "center", justifyContent: "center", @@ -119,6 +129,8 @@ export function AnnotationToolbar({ accessibilityRole="button" style={{ width: HIT_SIZE, + minWidth: MIN_HIT_SIZE, + flexShrink: 1, height: HIT_SIZE, alignItems: "center", justifyContent: "center", @@ -135,6 +147,8 @@ export function AnnotationToolbar({ accessibilityRole="button" style={{ width: HIT_SIZE, + minWidth: MIN_HIT_SIZE, + flexShrink: 1, height: HIT_SIZE, alignItems: "center", justifyContent: "center",