From 6022b23205fde0c555e13588ebcce32ac17e962b Mon Sep 17 00:00:00 2001 From: DNT_OF <3174600270@qq.com> Date: Sat, 7 Nov 2026 13:16:38 +0000 Subject: [PATCH 1/2] fix: keep write, erase, and undo atomic when input is interrupted A cancelled, blurred, or tool-switched stroke is dropped instead of committed. The eraser follows the pointer path in world units after zoom and pan. Undo restores the formula selection, and a gesture that does not move the sheet is not a history step. Handwriting recognition starts only from an explicit click. Co-authored-by: DNT_OF --- .github/workflows/package.yml | 5 + README.md | 11 +- package.json | 4 +- scripts/e2e-ink.ts | 306 ++++++++++++++++++++++++++++++++++ scripts/e2e-sheet.ts | 13 +- scripts/ink-geom-smoke.ts | 79 +++++++++ src/App.tsx | 39 +++-- src/board/Board.tsx | 121 +++++++++++--- src/board/InkFormula.tsx | 30 +++- src/board/history.ts | 58 +++++++ src/board/strokeGeom.ts | 72 ++++++-- src/board/useDocument.ts | 45 +++-- src/board/viewGeom.ts | 20 +++ src/i18n/messages.ts | 4 +- 14 files changed, 729 insertions(+), 78 deletions(-) create mode 100644 scripts/e2e-ink.ts create mode 100644 scripts/ink-geom-smoke.ts create mode 100644 src/board/history.ts create mode 100644 src/board/viewGeom.ts diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d4e83fd..2573dba 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -56,6 +56,7 @@ jobs: - run: npm run smoke:ocr - run: npm run smoke:ai - run: npm run test:pack + - run: npm run smoke:ink - name: Start the dev server run: | npm run dev > dev-server.log 2>&1 & @@ -73,6 +74,10 @@ jobs: env: SHOTS: e2e-shots run: npm run e2e:sheet + - name: Ink gestures + env: + SHOTS: e2e-shots + run: npm run e2e:ink - name: Per-formula plot end-to-end env: SHOTS: e2e-plots diff --git a/README.md b/README.md index a8cd253..9d1bd41 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ The dev server is `http://127.0.0.1:47321`. `npm run preview` serves the product | `npm run ocr:live` | 本机用仓库里的 CoMER INT8 读几笔样例。 / Run the bundled CoMER INT8 on a few sample strokes. | | `npm run smoke:ai` | 本地假接口上的连接测试、工具和 MCP。 / Connection test, tools, and MCP against a local fake endpoint. | | `npm run test:rust` | `crates/mathcore` 的 Rust 测试。 / Rust tests for `crates/mathcore`. | -| `npm run e2e` / `npm run e2e:sheet` | 需要先开着 dev。浏览器按 `CHROME_PATH`、Playwright 缓存、PATH，再按常见安装路径查找。 / Need a running dev server. The browser is `CHROME_PATH`, then the Playwright cache, then PATH, then common install paths. | +| `npm run smoke:ink` | 笔迹命中、缩放坐标和撤销栈。 / Ink hit testing, zoom coordinates, and the undo stack. | +| `npm run e2e` / `npm run e2e:sheet` / `npm run e2e:ink` | 需要先开着 dev。浏览器按 `CHROME_PATH`、Playwright 缓存、PATH，再按常见安装路径查找。`e2e:ink` 覆盖书写、擦除、撤销和中断。 / Need a running dev server. The browser is `CHROME_PATH`, then the Playwright cache, then PATH, then common install paths. `e2e:ink` covers writing, erasing, undo, and interrupted pointers. | | `npm run build:wasm` | 把 Rust 重新编成 `src/wasm/pkg`。 / Rebuild Rust into `src/wasm/pkg`. | ## 纸面、公式和坐标系 @@ -102,9 +103,9 @@ Switching language changes buttons, menus, dialogs, placeholders, hints, status ## Handwriting -默认识别是开的：笔迹停大约一秒，先判断像不像算式。普通句子留在纸上，不送进公式模型。像算式才走本机 CoMER。置信度够就写成公式并更新角落的图，纸上没有确认条。按 R 会对最近的笔迹再认一次。设置里可以关掉自动识别。 +手写识别要点一下才开始：按 R，或点工具条上的识别。抬笔不会自动识别。先判断像不像算式。普通句子留在纸上，不送进公式模型。像算式才走本机 CoMER。置信度够就写成公式并更新角落的图，纸上没有确认条。设置里可以打开“写完稍停再认”。 -Recognition is on by default. About a second after the ink settles, a text gate decides whether it looks like a formula. Ordinary writing stays on the paper and is not sent to the formula model. Formula-like ink goes to the local CoMER model. A confident reading is written as a formula and the corner plot updates. There is no confirmation strip. R recognizes the latest ink again. Automatic recognition can be turned off in Settings. +Handwriting is recognized only when you ask: press R, or use the recognize button. Lifting the pen does not recognize it. A text gate decides whether it looks like a formula. Ordinary writing stays on the paper and is not sent to the formula model. Formula-like ink goes to the local CoMER model. A confident reading is written as a formula and the corner plot updates. There is no confirmation strip. Settings can turn on recognition after a short pause. 模型是 CoMER（ECCV 2022）的 INT8 ONNX，权重由 [ink-on](https://github.com/kimseungdae/ink-on) 以 Apache-2.0 分发。它能读一些清楚的短式，不能当作任意手写的完整识别器：认不出就保持笔迹，不会用 `y = x` 充数。单独的 `sigma` 是符号，求和与积分留在纸上，不因此打开坐标系。 @@ -176,9 +177,9 @@ iOS 最低系统是 16.4。能装到别人手机上的 IPA 需要 Apple Develope The iOS minimum is 16.4. An IPA that installs on someone else's phone needs an Apple Developer certificate and profile: `IOS_CERTIFICATE`, `IOS_CERTIFICATE_PASSWORD`, `IOS_MOBILE_PROVISION`, or `APPLE_API_ISSUER`, `APPLE_API_KEY`, `APPLE_API_KEY_PATH`. This repository does not enroll a developer account, does not upload to TestFlight or the App Store, and has not been installed on an iPhone. -`.github/workflows/package.yml` 在推送到 `main` 或 `cursor/mathscrap-packaging-*` 时，以及指向 `main` 的 pull request 上运行。`test` 作业没有额外的跳过条件：这三种情况都会在 `ubuntu-latest` 上跑 `npm run lint`、`npm run test:rust`、`npm run smoke`、`npm run smoke:ocr`、`npm run smoke:ai`、`npm run test:pack`，然后拉起 dev 服务器，依次跑 `npm run e2e:sheet` 和 `npm run e2e`（逐条作图）。失败时上传 `e2e-shots`、`e2e-plots` 和 `dev-server.log`。这些步骤不需要密钥。网页、debug APK 和未签名 Windows 安装程序仍只在 push 上、并且 `test` 通过之后构建；pull request 不打安装包。`.github/workflows/package-ios.yml` 只在手动启动时使用 macOS runner。这些 workflow 不创建 GitHub Release。本文件不提供下载地址。 +`.github/workflows/package.yml` 在推送到 `main` 或 `cursor/mathscrap-packaging-*` 时，以及指向 `main` 的 pull request 上运行。`test` 作业没有额外的跳过条件：这三种情况都会在 `ubuntu-latest` 上跑 `npm run lint`、`npm run test:rust`、`npm run smoke`、`npm run smoke:ocr`、`npm run smoke:ai`、`npm run test:pack`、`npm run smoke:ink`，然后拉起 dev 服务器，依次跑 `npm run e2e:sheet`、`npm run e2e:ink` 和 `npm run e2e`（逐条作图）。失败时上传 `e2e-shots`、`e2e-plots` 和 `dev-server.log`。这些步骤不需要密钥。网页、debug APK 和未签名 Windows 安装程序仍只在 push 上、并且 `test` 通过之后构建；pull request 不打安装包。`.github/workflows/package-ios.yml` 只在手动启动时使用 macOS runner。这些 workflow 不创建 GitHub Release。本文件不提供下载地址。 -`.github/workflows/package.yml` runs on a push to `main` or `cursor/mathscrap-packaging-*`, and on a pull request into `main`. The `test` job has no extra skip condition, so all three of those runs execute it on `ubuntu-latest`: `npm run lint`, `npm run test:rust`, `npm run smoke`, `npm run smoke:ocr`, `npm run smoke:ai`, `npm run test:pack`, then a dev server, then `npm run e2e:sheet` and `npm run e2e` (one plot per formula). On failure it uploads `e2e-shots`, `e2e-plots`, and `dev-server.log`. Those steps need no secrets. The web bundle, debug APK, and unsigned Windows installer still build only on a push, and only after `test` passes. A pull request does not build installers. `.github/workflows/package-ios.yml` uses a macOS runner only when started by hand. These workflows do not create a GitHub Release. This file does not publish a download URL. +`.github/workflows/package.yml` runs on a push to `main` or `cursor/mathscrap-packaging-*`, and on a pull request into `main`. The `test` job has no extra skip condition, so all three of those runs execute it on `ubuntu-latest`: `npm run lint`, `npm run test:rust`, `npm run smoke`, `npm run smoke:ocr`, `npm run smoke:ai`, `npm run test:pack`, `npm run smoke:ink`, then a dev server, then `npm run e2e:sheet`, `npm run e2e:ink`, and `npm run e2e` (one plot per formula). On failure it uploads `e2e-shots`, `e2e-plots`, and `dev-server.log`. Those steps need no secrets. The web bundle, debug APK, and unsigned Windows installer still build only on a push, and only after `test` passes. A pull request does not build installers. `.github/workflows/package-ios.yml` uses a macOS runner only when started by hand. These workflows do not create a GitHub Release. This file does not publish a download URL. ## 许可和作者 diff --git a/package.json b/package.json index b4025df..605bcce 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,10 @@ "smoke:ocr": "tsx scripts/ocr-smoke.ts", "ocr:live": "tsx scripts/ocr-live.ts", "mcp": "tsx mcp/server.ts", + "smoke:ink": "tsx scripts/ink-geom-smoke.ts", "e2e": "tsx scripts/e2e-plots.ts", - "e2e:sheet": "tsx scripts/e2e-sheet.ts" + "e2e:sheet": "tsx scripts/e2e-sheet.ts", + "e2e:ink": "tsx scripts/e2e-ink.ts" }, "dependencies": { "@tauri-apps/api": "2.12.4", diff --git a/scripts/e2e-ink.ts b/scripts/e2e-ink.ts new file mode 100644 index 0000000..b9b0262 --- /dev/null +++ b/scripts/e2e-ink.ts @@ -0,0 +1,306 @@ +/** + * Write → partial erase → undo/redo → keep writing, including cancelled pointers. + * + * npm run dev + * npm run e2e:ink + * + * Pointers here are dispatched by Playwright (mouse) or by synthetic PointerEvents + * (pen, touch, cancel, lost capture). That is not a physical stylus, palm, or device. + */ +import { mkdirSync } from 'node:fs' +import { chromium, type Page } from 'playwright-core' +import { resolveBrowser } from './browser-path.ts' + +const URL = process.env.APP_URL ?? 'http://127.0.0.1:47321/' +const BROWSER = resolveBrowser() +console.log(`browser ${BROWSER.executablePath} (${BROWSER.via})`) +const SHOTS = process.env.SHOTS ?? '/tmp/mathscrap-e2e' +mkdirSync(SHOTS, { recursive: true }) + +const problems: string[] = [] +const check = (ok: boolean, what: string) => { + console.log(`${ok ? 'ok ' : 'FAIL'} ${what}`) + if (!ok) problems.push(what) +} + +interface InkState { + total: number + paths: number + scale: number + vx: number + vy: number + left: number + top: number + strokes: { pts: number[] }[] +} + +async function ink(page: Page): Promise { + return page.evaluate(() => { + const read = (window as unknown as { mathscrap: { tools: { call: (n: string, a: object) => InkState } } }).mathscrap.tools.call('ink_read', {}) + const board = document.querySelector('.kraft-page') as HTMLElement + const world = [...board.children].find((c) => (c as HTMLElement).style.transform.includes('scale')) as HTMLElement + const m = new DOMMatrixReadOnly(getComputedStyle(world).transform) + const rect = board.getBoundingClientRect() + return { + total: read.total, + strokes: read.strokes, + paths: board.querySelectorAll('svg[data-ink] path').length, + scale: m.a, + vx: m.e, + vy: m.f, + left: rect.left, + top: rect.top, + } + }) +} + +function worldToClient(s: InkState, x: number, y: number) { + return { x: s.left + s.vx + x * s.scale, y: s.top + s.vy + y * s.scale } +} + +async function raw(page: Page, phase: 'down' | 'move' | 'up' | 'cancel' | 'lost', x: number, y: number, pointerId: number, pointerType: 'mouse' | 'pen' | 'touch') { + await page.evaluate( + ({ phase, x, y, pointerId, pointerType }) => { + const el = document.querySelector('.kraft-page')! + const name = phase === 'down' ? 'pointerdown' : phase === 'move' ? 'pointermove' : phase === 'up' ? 'pointerup' : phase === 'cancel' ? 'pointercancel' : 'lostpointercapture' + el.dispatchEvent( + new PointerEvent(name, { + bubbles: true, + cancelable: true, + pointerId, + pointerType, + clientX: x, + clientY: y, + button: 0, + buttons: phase === 'up' || phase === 'cancel' ? 0 : 1, + pressure: pointerType === 'pen' ? 0.55 : 0.5, + isPrimary: true, + }), + ) + }, + { phase, x, y, pointerId, pointerType }, + ) +} + +async function seed(page: Page) { + await page.addInitScript(() => { + localStorage.setItem('mathscrap.lang', 'zh') + localStorage.setItem('mathscrap.prefs.v2', JSON.stringify({ penOnly: false, showGrid: false, showHints: false, autoOcr: false })) + localStorage.removeItem('mathscrap.doc.v2') + }) +} + +async function boot(page: Page) { + const errors: string[] = [] + page.on('pageerror', (e) => errors.push(e.message)) + await page.goto(URL, { waitUntil: 'networkidle' }) + await page.waitForFunction(() => !!(window as unknown as { mathscrap?: unknown }).mathscrap) + await page.keyboard.press('p') + return errors +} + +/** A short stroke near the middle of the sheet. Returns when it has been committed. */ +async function drawMouse(page: Page, x: number, y: number) { + await page.keyboard.press('p') + await page.mouse.move(x, y) + await page.mouse.down() + for (let i = 1; i <= 6; i++) await page.mouse.move(x + i * 16, y + Math.sin(i) * 8) + await page.mouse.up() + await page.waitForTimeout(40) +} + +async function gestureLoop(page: Page, label: string) { + const origin = await ink(page) + const x = Math.round(origin.left + origin.vx + 180) + const y = Math.round(origin.top + 220) + await drawMouse(page, x, y) + let s = await ink(page) + check(s.total === origin.total + 1, `${label}: mouse stroke commits`) + + await page.keyboard.press('Control+z') + s = await ink(page) + check(s.total === origin.total, `${label}: undo removes that stroke`) + await page.keyboard.press('Control+Shift+z') + s = await ink(page) + check(s.total === origin.total + 1, `${label}: redo puts it back`) + + const before = (await ink(page)).total + await page.mouse.move(x, y + 70) + await page.mouse.down() + await page.mouse.move(x + 90, y + 90) + await raw(page, 'cancel', x + 90, y + 90, 1, 'mouse') + await page.mouse.up() + s = await ink(page) + check(s.total === before && s.paths === before, `${label}: mouse pointercancel leaves no stroke and no ghost path`) + + for (const kind of ['pen', 'touch'] as const) { + const n = (await ink(page)).total + const id = kind === 'pen' ? 7 : 8 + await raw(page, 'down', x + 20, y + 120, id, kind) + await raw(page, 'move', x + 80, y + 140, id, kind) + await raw(page, 'cancel', x + 80, y + 140, id, kind) + s = await ink(page) + check(s.total === n && s.paths === n, `${label}: ${kind} pointercancel leaves no stroke`) + } + + { + const n = (await ink(page)).total + await raw(page, 'down', x + 30, y + 160, 9, 'pen') + await raw(page, 'move', x + 100, y + 170, 9, 'pen') + await raw(page, 'lost', x + 100, y + 170, 9, 'pen') + await raw(page, 'move', x + 130, y + 180, 9, 'pen') + s = await ink(page) + check(s.total === n && s.paths === n, `${label}: lost pointer capture drops the stroke and ignores later moves`) + } + + { + const n = (await ink(page)).total + await page.mouse.move(x, y + 200) + await page.mouse.down() + await page.mouse.move(x + 70, y + 210) + await page.keyboard.press('e') + await page.waitForFunction(() => document.querySelector('[aria-label="橡皮（整笔擦除）"]')?.getAttribute('aria-pressed') === 'true') + await page.mouse.up() + s = await ink(page) + check(s.total === n && s.paths === n, `${label}: switching tool mid-stroke does not commit it`) + await page.keyboard.press('p') + } + + { + const n = (await ink(page)).total + await page.mouse.move(x + 10, y + 240) + await page.mouse.down() + await page.mouse.move(x + 80, y + 250) + await page.evaluate(() => window.dispatchEvent(new Event('blur'))) + await page.mouse.up() + s = await ink(page) + check(s.total === n && s.paths === n, `${label}: blur aborts the in-progress stroke`) + } + + await drawMouse(page, x, y + 280) + s = await ink(page) + check(s.total === before + 1, `${label}: a later stroke still commits after the interruptions`) +} + +async function eraseAtZoom(page: Page, label: string) { + await page.keyboard.press('p') + const board = await page.locator('.kraft-page').boundingBox() + if (!board) throw new Error('board missing') + const x = Math.round(board.x + board.width * 0.45) + const y = Math.round(board.y + board.height * 0.42) + await drawMouse(page, x, y) + let s = await ink(page) + const stroke = s.strokes[s.strokes.length - 1] + const xs: number[] = [] + const ys: number[] = [] + for (let i = 0; i < stroke.pts.length; i += 3) { + xs.push(stroke.pts[i]) + ys.push(stroke.pts[i + 1]) + } + const minX = Math.min(...xs) + const maxX = Math.max(...xs) + const midY = ys[Math.floor(ys.length / 2)] + + await page.getByRole('button', { name: '更多' }).click() + await page.getByRole('button', { name: '放大' }).click() + await page.keyboard.press('Escape') + s = await ink(page) + check(s.scale > 1, `${label}: zoom in changes the view scale (${s.scale.toFixed(3)})`) + + // Endpoints sit well outside the stroke. Only the segment between them crosses it. + const farL = worldToClient(s, minX - 80, midY) + const farR = worldToClient(s, maxX + 80, midY) + await page.keyboard.press('e') + await page.mouse.move(farL.x, farL.y) + await page.mouse.down() + await page.mouse.move(farR.x, farR.y) + await page.mouse.up() + const after = await ink(page) + check(after.total === s.total - 1, `${label}: one eraser swipe across the stroke at zoom ${s.scale.toFixed(2)} removes it`) + + await page.keyboard.press('Control+z') + check((await ink(page)).total === s.total, `${label}: undo restores the erased stroke`) + await page.keyboard.press('Control+Shift+z') + check((await ink(page)).total === s.total - 1, `${label}: redo removes it again`) + + // A miss must not consume the undo slot that brings the stroke back. + const miss = worldToClient(await ink(page), minX - 160, midY - 160) + await page.mouse.move(miss.x, miss.y) + await page.mouse.down() + await page.mouse.up() + await page.keyboard.press('Control+z') + check((await ink(page)).total === s.total, `${label}: an eraser miss does not take an undo step`) + await page.keyboard.press('p') +} + +async function formulaHistory(page: Page, label: string) { + await page.keyboard.press('v') + await page.keyboard.press('/') + const ta = page.getByRole('textbox', { name: '在纸上写公式' }) + await ta.waitFor({ state: 'visible' }) + await ta.fill('y = x') + await page.keyboard.press('Enter') + await ta.waitFor({ state: 'detached' }) + const formula = page.locator('[data-formula]') + await formula.click() + const selected = async () => (await formula.locator('.ink-formula').getAttribute('class'))?.includes('border-dashed') ?? false + check(await selected(), `${label}: clicking a formula selects it`) + const start = await formula.getAttribute('style') + await page.keyboard.press('Delete') + check((await formula.count()) === 0, `${label}: delete removes the formula`) + await page.keyboard.press('Control+z') + check((await formula.count()) === 1 && (await selected()), `${label}: undo restores the formula and its selection`) + + await page.keyboard.press('v') + const box = await formula.boundingBox() + if (!box) throw new Error('formula box missing') + const px = box.x + 24 + const py = box.y + 16 + await page.mouse.move(px, py) + await page.mouse.down() + await page.mouse.move(px + 36, py + 28) + await page.mouse.move(px, py) + await page.mouse.up() + await page.keyboard.press('Control+z') + check((await formula.count()) === 0, `${label}: dragging a formula back to its start does not add an undo step`) + + await page.keyboard.press('Control+Shift+z') + check((await formula.count()) === 1, `${label}: redo brings the formula back`) + const again = await formula.boundingBox() + if (!again) throw new Error('formula box missing after redo') + await page.mouse.move(again.x + 24, again.y + 16) + await page.mouse.down() + await page.mouse.move(again.x + 70, again.y + 40) + await page.evaluate(() => { + const el = document.querySelector('[data-formula]')! + el.dispatchEvent(new PointerEvent('pointercancel', { bubbles: true, cancelable: true, pointerId: 1, pointerType: 'mouse', clientX: 0, clientY: 0, button: 0, buttons: 0, isPrimary: true })) + }) + await page.mouse.up() + const style = await formula.getAttribute('style') + check(style === start, `${label}: pointercancel rolls a formula drag back (${style} vs ${start})`) + await page.keyboard.press('Control+z') + check((await formula.count()) === 0, `${label}: a cancelled drag is not an undo step`) +} + +async function run(viewport: { width: number; height: number }, dpr: number, label: string, shot: string) { + const browser = await chromium.launch({ executablePath: BROWSER.executablePath, args: ['--no-sandbox', '--use-gl=swiftshader'] }) + const page = await browser.newPage({ viewport, deviceScaleFactor: dpr }) + await seed(page) + const errors = await boot(page) + await gestureLoop(page, label) + await eraseAtZoom(page, label) + await formulaHistory(page, label) + check(errors.length === 0, `${label}: no page errors (${errors.join(' | ')})`) + await page.screenshot({ path: `${SHOTS}/${shot}.png` }) + await browser.close() +} + +await run({ width: 1440, height: 900 }, 1, 'desktop', 'ink-desktop') +await run({ width: 390, height: 844 }, 1, 'phone', 'ink-phone') +await run({ width: 1280, height: 800 }, 2, 'dpr2', 'ink-dpr2') + +if (problems.length) { + console.log(`\n${problems.length} failed`) + process.exit(1) +} +console.log('\nink e2e passed') diff --git a/scripts/e2e-sheet.ts b/scripts/e2e-sheet.ts index 6c63648..15b5541 100644 --- a/scripts/e2e-sheet.ts +++ b/scripts/e2e-sheet.ts @@ -290,7 +290,8 @@ async function dockAndLive() { } check(seen.length >= 3 && seen[seen.length - 1] >= 3, `reveal: statements emerge one by one (${seen.join(' → ')})`) - // 5. stray ink is not turned into a formula and nothing is painted over the paper + // 5. stray ink is not turned into a formula and nothing is painted over the paper. + // Recognition starts only from the button or R, not when the pen lifts. await page2.keyboard.press('p') await page2.mouse.move(320, 320) await page2.mouse.down() @@ -301,13 +302,15 @@ async function dockAndLive() { await page2.mouse.down() await page2.mouse.move(540, 355, { steps: 8 }) await page2.mouse.up() - await page2.waitForFunction(() => document.documentElement.dataset.ocr === 'run', null, { timeout: 5000 }) + await page2.waitForTimeout(1200) + check((await page2.evaluate(() => document.documentElement.dataset.ocr ?? 'idle')) !== 'run', 'ocr: lifting the pen does not start recognition') + await page2.keyboard.press('r') await page2.waitForFunction(() => document.documentElement.dataset.ocr === 'idle', null, { timeout: 25000 }) check((await page2.locator('[data-ink-chip]').count()) === 0 && (await page2.getByText('手写 → 公式').count()) === 0, 'ocr: no confirm chip or dialog covers the paper') check((await page2.locator('[data-formula]').count()) === 1, 'ocr: unrecognized ink is not written onto the sheet') await page2.screenshot({ path: `${SHOTS}/dock-ocr-chip.png` }) - // 6. confident handwriting is written by itself and the corner plot opens + // 6. an explicit recognize click writes confident handwriting and opens the corner plot const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }) const page3 = await ctx.newPage() page3.on('pageerror', (e) => errors.push(e.message)) @@ -325,6 +328,10 @@ async function dockAndLive() { localStorage.setItem('mathscrap.doc.v2', saved) }, JSON.stringify(doc)) await page3.goto(RAW, { waitUntil: 'networkidle' }) + await page3.waitForFunction(() => !!(window as unknown as { mathscrap?: unknown }).mathscrap) + await page3.waitForTimeout(400) + check((await page3.evaluate(() => document.documentElement.dataset.ocr ?? 'idle')) !== 'run', 'ocr: saved ink is not recognized on load') + await page3.keyboard.press('r') await page3.waitForFunction(() => document.documentElement.dataset.ocr === 'idle', null, { timeout: 60000 }) const read = await page3.locator('[data-formula]').evaluateAll((els) => els.map((el) => el.getAttribute('data-formula-text') ?? '')) const vecs = read.filter((t) => /^\\vec\{[a-z]\} = \(/.test(t)) diff --git a/scripts/ink-geom-smoke.ts b/scripts/ink-geom-smoke.ts new file mode 100644 index 0000000..80e7719 --- /dev/null +++ b/scripts/ink-geom-smoke.ts @@ -0,0 +1,79 @@ +/** + * Screen/world hit testing and undo transactions. + * Run: npm run smoke:ink + * + * The eraser radius is MathScrap's own CSS-pixel reach. Device pixel ratio is not + * a term in the conversion: client coordinates and the sheet transform are both CSS pixels. + */ +import assert from 'node:assert/strict' +import { applyRedo, applyUndo, pushCommit, sameDoc } from '../src/board/history.ts' +import { hitStroke, hitStrokeCapsule } from '../src/board/strokeGeom.ts' +import { blankDoc } from '../src/board/useDocument.ts' +import { clientToWorld, eraserWorldRadius, ERASER_SCREEN_PX } from '../src/board/viewGeom.ts' +import type { Stroke } from '../src/board/types.ts' + +const ink = (pts: number[], width = 2): Stroke => ({ id: 's', pts, color: '#2b2118', width, style: 'ink', createdAt: 1 }) + +const view = { x: 30, y: 40, scale: 2 } +const origin = { left: 10, top: 20 } +const client = { x: origin.left + view.x + 100 * view.scale, y: origin.top + view.y + 80 * view.scale } +const world = clientToWorld(client.x, client.y, origin, view) +assert.deepEqual(world, { x: 100, y: 80 }) +assert.deepEqual(clientToWorld(client.x, client.y, origin, view), world) + +const radius = eraserWorldRadius(view.scale) +assert.equal(radius, ERASER_SCREEN_PX / view.scale) +const line = ink([100, 100, 0.5, 160, 100, 0.5]) +assert.equal(hitStroke(line, 100, 105, radius), true) +assert.equal(hitStroke(line, 100, 130, radius), false) +assert.equal(hitStroke(line, 40, 100, radius), false) +assert.equal(hitStroke(line, 220, 100, radius), false) +assert.equal(hitStrokeCapsule(line, 40, 100, 220, 100, radius), true) + +const upright = ink([0, 0, 0.5, 0, 200, 0.5]) +assert.equal(hitStroke(upright, -80, 100, 12), false) +assert.equal(hitStroke(upright, 80, 100, 12), false) +assert.equal(hitStrokeCapsule(upright, -80, 100, 80, 100, 12), true) + +const empty = blankDoc() +const card = { id: 'a', x: 10, y: 20, text: 'y = x' } +const placed = { ...empty, cards: [card] } +const samePlace = { ...empty, cards: [{ ...card }] } +const moved = { ...empty, cards: [{ ...card, x: 18 }] } +assert.equal(sameDoc(placed, samePlace), true) +assert.equal(sameDoc(placed, moved), false) + +const untouched = pushCommit([], placed, samePlace, 'a') +assert.equal(untouched.changed, false) +assert.equal(untouched.past.length, 0) + +const recorded = pushCommit([], placed, moved, 'a') +assert.equal(recorded.changed, true) +assert.equal(recorded.past[0]?.selected, 'a') +const undone = applyUndo(recorded.past, [], moved, null) +assert.equal(undone.changed, true) +assert.equal(undone.doc, placed) +assert.equal(undone.selected, 'a') +const redone = applyRedo(undone.past, undone.future, undone.doc, undone.selected) +assert.equal(redone.doc, moved) +assert.equal(redone.selected, null) +assert.equal(applyUndo([], [], empty, null).changed, false) + +let past = recorded.past +let prev = moved +for (let i = 0; i < 130; i++) { + const next = { ...empty, cards: [{ ...card, x: 100 + i }] } + const step = pushCommit(past, prev, next, null) + assert.equal(step.changed, true) + past = step.past + prev = next +} +assert.equal(past.length, 120) + +const withStroke = { ...empty, strokes: [line] } +const erased = { ...empty, strokes: [] as Stroke[] } +const erase = pushCommit([], withStroke, erased, null) +assert.equal(erase.changed, true) +assert.equal(pushCommit(erase.past, erased, { ...erased, strokes: [] }, null).changed, false) + +console.log('ink geom smoke passed') diff --git a/src/App.tsx b/src/App.tsx index 34d570a..63cffab 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -30,7 +30,7 @@ import { PlotPanel } from '@/plot/PlotPanel' import { PlotPane } from '@/plot/PlotPane' const PREFS_KEY = 'mathscrap.prefs.v2' -const DEFAULT_PREFS: Prefs = { penOnly: false, showGrid: false, showHints: true, autoOcr: true } +const DEFAULT_PREFS: Prefs = { penOnly: false, showGrid: false, showHints: true, autoOcr: false } /** vertical rhythm of formulas written one under another, like lines in a notebook */ const LINE = 60 @@ -120,12 +120,11 @@ interface ComposerState { export default function App() { const { t } = useI18n() const compact = useMediaQuery('(max-width: 767px)') - const { doc, docRef, commit, preview, undo, redo, canUndo, canRedo } = useDocument() + const { doc, docRef, commit, preview, undo, redo, canUndo, canRedo, selected, setSelected } = useDocument() const [view, setView] = useState(() => fitView(doc, window.innerWidth < 768)) const [tool, setTool] = useState('pen') const [pen, setPen] = useState({ color: '#2b2118', width: 3, style: 'ink' }) const [prefs, setPrefsState] = useState(loadPrefs) - const [selected, setSelected] = useState(null) /** `open` shows the corner plot. `details` expands the readouts. `full` is the large dialog. */ const [plot, setPlot] = useState<{ open: boolean; details: boolean; full: boolean; focusId: string | null }>({ open: false, details: false, full: false, focusId: null }) const [composerText, setComposerText] = useState('') @@ -208,7 +207,7 @@ export default function App() { setSelected(null) toast(t('clearedOne'), { action: { label: t('undo'), onClick: undo } }) }, - [commit, t, undo], + [commit, setSelected, t, undo], ) const closeComposer = () => { @@ -237,7 +236,7 @@ export default function App() { setComposerText('') setComposer({ open: true, at: { x, y }, editingId: null }) }, - [], + [setSelected], ) const editCard = useCallback( @@ -288,12 +287,14 @@ export default function App() { ) const recognize = useCallback(() => { - const pool = unmarkedInk(docRef).slice(-24) + const pool = unmarkedInk(docRef) if (!pool.length) { toast(t('writeFirst')) return } const clusters = clusterStrokes(pool) + if (!clusters.length) return + document.documentElement.dataset.ocr = 'run' let pending = clusters.length let wrote = 0 for (const cl of clusters) { @@ -303,7 +304,10 @@ export default function App() { const best = cands[0] if (best && best.confidence >= OCR_ACCEPT && acceptInk(best.text, cl)) wrote += 1 pending -= 1 - if (pending === 0 && wrote === 0) toast(t('notRecognized')) + if (pending === 0) { + document.documentElement.dataset.ocr = 'idle' + if (wrote === 0) toast(t('notRecognized')) + } }) } }, [acceptInk, docRef, t]) @@ -394,7 +398,7 @@ export default function App() { } window.addEventListener('keydown', onKey) return () => window.removeEventListener('keydown', onKey) - }, [undo, redo, plot.open, plot.full, recognize, selected, deleteCard, openPlot, closePlot, togglePlot, placeComposer, writingSpot]) + }, [undo, redo, plot.open, plot.full, recognize, selected, setSelected, deleteCard, openPlot, closePlot, togglePlot, placeComposer, writingSpot]) // MCP-ready local tool bridge for experimentation from devtools. // getDoc reads the React state captured for this render, so the bridge is rebuilt when the sheet changes. @@ -449,7 +453,7 @@ export default function App() { toast(e instanceof Error ? e.message : t('importFailed')) } }, - [commit, t, undo], + [commit, setSelected, t, undo], ) const anchor = composer.at ? { left: composer.at.x * view.scale + view.x, top: composer.at.y * view.scale + view.y } : null @@ -473,10 +477,15 @@ export default function App() { editingCard={composer.editingId} onSelectCard={setSelected} onPlace={placeComposer} - onAddStroke={(s) => commit((d) => ({ ...d, strokes: [...d.strokes, s] }))} + onAddStroke={(s) => { + commit((d) => ({ ...d, strokes: [...d.strokes, s] })) + setSelected(null) + }} onErase={(ids) => { - const set = new Set(ids) - commit((d) => ({ ...d, strokes: d.strokes.filter((s) => !set.has(s.id)) })) + const gone = new Set(ids) + const prev = docRef.current + commit((d) => ({ ...d, strokes: d.strokes.filter((s) => !gone.has(s.id)) })) + if (docRef.current !== prev) setSelected(null) }} onCardDragStart={() => (dragBase.current = docRef.current)} onCardDrag={(id, dx, dy) => preview((d) => ({ ...d, cards: d.cards.map((c) => (c.id === id ? { ...c, x: c.x + dx, y: c.y + dy } : c)) }))} @@ -484,6 +493,11 @@ export default function App() { if (dragBase.current) commit(docRef.current, dragBase.current) dragBase.current = null }} + onCardDragCancel={() => { + const base = dragBase.current + dragBase.current = null + if (base) preview(() => base) + }} onPlot={openPlot} onEdit={editCard} onDelete={deleteCard} @@ -595,6 +609,7 @@ export default function App() {