diff --git a/docs/overlay.md b/docs/overlay.md index 56afe3e..1585c52 100644 --- a/docs/overlay.md +++ b/docs/overlay.md @@ -72,6 +72,10 @@ The settings choose what an answer's words are given as (`captions`: `both`, `te - **Text only.** No voice is asked for and nothing is spoken: the step's whole sentence is shown at once and the step lasts as long as it takes to read it (`readingSeconds`: three words a second and a moment to take the page in, at least 3.5 s), and the drawing is timed to that as it is to a voice's length. - **Voice only.** No words on screen. +## When the window is on the screen + +The overlay is one transparent, topmost window per display, so a window left up all day sits over everything the person watches (a playing video included) whether or not anything is drawn in it. It is therefore hidden while it has nothing to show, and shown without taking focus only while an answer is asked for, written or shown (`src/main/overlay-visibility.ts`). Any message that begins something to see or hear (the cue of a question, the first step, a notice, the pen going on) raises the window in the same moment, before the page is told, and for a few seconds after it whatever the page has said; after that the page's own list of what it shows (the visible regions: the model, marks, diagrams, captions, answer card, pills, a guide target) decides, together with the pen, a held click, and, for a window already up, an answer being asked or written. It goes a moment (under a second) after the last of those, so a pause between steps or a fade never makes it blink. A busy answer never raises a window by itself, so a second display stays down. A development run with a backdrop or a fixed window size keeps the window up as before, and a run that keeps windows hidden never shows them. The page keeps running while the window is down (`backgroundThrottling` is off). The hotkey, the guide, the pins and the click guard do not depend on the window being up: they read the page's regions, which are empty exactly when nothing is shown. + ## Fading When a turn has finished, the scene stays at full strength for `FADE_AFTER_MS` (set in `src/shared/ipc.ts`), then fades over 1.2 s, unless the pointer is on the model or the app's words; touching it starts the count again. A faded scene takes no clicks. It is hidden, not cleared, so a follow-up still reaches its ids; the next question or Clear removes it. `fade.ts` is the logic and is tested without a clock. diff --git a/package-lock.json b/package-lock.json index 4bfb572..dc65419 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "talkthrough", - "version": "0.7.0", + "version": "0.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "talkthrough", - "version": "0.7.0", + "version": "0.7.1", "license": "MIT", "dependencies": { "get-windows": "10.0.0", diff --git a/package.json b/package.json index 18ea622..c78d340 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "talkthrough", - "version": "0.7.0", + "version": "0.7.1", "description": "An AI that talks you through anything on your screen, and draws as it talks.", "keywords": [ "screen assistant", diff --git a/src/main/overlay-visibility.ts b/src/main/overlay-visibility.ts new file mode 100644 index 0000000..4f05046 --- /dev/null +++ b/src/main/overlay-visibility.ts @@ -0,0 +1,91 @@ +// When an overlay window is on the screen. The overlay is a full-screen, transparent, topmost window; left up all day it sits over everything the person +// watches, whether or not anything is drawn in it. So it is hidden while it has nothing to show and shown (without taking focus) only while an answer is +// asked for, written or shown. This holds the rule, with time injected; overlay-windows.ts feeds it what it knows and does the showing. + +/** After the last thing that needed the window, how long it stays before it goes (so a pause between steps, or a fade, never makes it blink). */ +export const HIDE_AFTER_MS = 800; +/** After a message that starts something (a question, a step, a notice), the window is kept this long even if the page has not yet said what it draws. */ +export const WAKE_MS = 3000; + +export type OverlayVisibilityDeps = { + show(): void; + hide(): void; + now?: () => number; + hideAfterMs?: number; + wakeMs?: number; +}; + +export type OverlayVisibility = { + /** Something is about to be drawn or said (a message for the page): the window is up before the page gets it. */ + wake(): void; + /** What is known now: whether anything needs the window. Call it when that may have changed, and now and then. */ + update(occupied: boolean): void; + readonly shown: boolean; +}; + +export function createOverlayVisibility(deps: OverlayVisibilityDeps): OverlayVisibility { + const now = deps.now ?? Date.now; + const hideAfter = deps.hideAfterMs ?? HIDE_AFTER_MS; + const wakeFor = deps.wakeMs ?? WAKE_MS; + let shown = false; + /** The last moment the window was known to be needed. */ + let needed = -Infinity; + const up = (): void => { + if (shown) return; + shown = true; + deps.show(); + }; + return { + wake(): void { + needed = Math.max(needed, now() + wakeFor); + up(); + }, + update(occupied: boolean): void { + const t = now(); + if (occupied) { + needed = Math.max(needed, t); + up(); + } else if (shown && t - needed >= hideAfter) { + shown = false; + deps.hide(); + } + }, + get shown(): boolean { return shown; }, + }; +} + +/** The messages to a page that begin something to see or hear. The rest (clear, dismiss, the end of a turn, an idle status, queries about the pointer) only matter when the window is already up. */ +export function wakesOverlay(message: { kind: string; status?: string; on?: boolean }): boolean { + switch (message.kind) { + case 'play': case 'begin': case 'step': case 'step-open': case 'action': case 'step-close': case 'notice': return true; + case 'status': return message.status !== 'idle'; + case 'pen': return message.on === true; + default: return false; + } +} + +export type OccupancyState = { + /** A development run that keeps the window up (a backdrop or a fixed size to look at). */ + alwaysUp: boolean; + pen: boolean; + /** The page says a drawing of an answer is up. */ + showing: boolean; + /** The page holds a click lease, a press on the model, or the pointer button. */ + holding: boolean; + /** An answer is being asked for, written or heard (the session says; it is not tied to one display). */ + busy: boolean; + /** This window is already up, so the busy answer is plainly on this display. */ + shown: boolean; + /** The alpha of every region the page last published (pills, captions, answer cards, marks, the model). */ + alphas: readonly number[]; +}; + +/** + * Whether anything needs the window. The page's own list of what it shows decides what is drawn; the rest keep a window that is up while the person is + * in the middle of something. A busy answer never raises a window by itself (it would light every display): the message that started it does. + */ +export function overlayOccupied(s: OccupancyState): boolean { + if (s.alwaysUp || s.pen || s.showing || s.holding) return true; + if (s.busy && s.shown) return true; + return s.alphas.some((a) => a > 0.02); +} diff --git a/src/main/overlay-windows.ts b/src/main/overlay-windows.ts index b19bac6..85dfaa0 100644 --- a/src/main/overlay-windows.ts +++ b/src/main/overlay-windows.ts @@ -8,6 +8,7 @@ import type { ButtonCatchers, ButtonRect } from './button-catchers.ts'; import { electronCatcherWindow } from './button-window.ts'; import { createClickGuard, createGuideArmGate, createKeyGate, createKeyHold, ignoreMouse, keyWanted } from './interaction.ts'; import { createEscape } from './escape.ts'; +import { createOverlayVisibility, overlayOccupied, wakesOverlay } from './overlay-visibility.ts'; import type { Region } from './interaction.ts'; import { containsWireContentExceptModel, containsWireId, containsWireRegion, createVisibleRegionReceiver } from './visible-region-wire.ts'; import type { VoiceBridge } from './voice-bridge.ts'; @@ -71,7 +72,7 @@ export type OverlayOptions = { onDetail?: (displayId: number) => void; }; -type Overlay = { catchers: ButtonCatchers; buttons: ButtonRect[]; visible: ReturnType; guide: ReturnType; modelHit?: { x: number; y: number; revision: number; askedAt: number; hit?: boolean }; window: BrowserWindow; display: Display; ready: boolean; queue: SceneMessage[]; pageWantsClicks: boolean; showing: boolean; pointerHeld: boolean; mouseIgnored: boolean; pen: boolean; penOffAt: number; /** The person pressed on the model (E is then ours until they leave it). */ engaged: boolean; /** The model's region on the screen (what the page last said the pointer is over) and the gate that checks the real cursor against it. */ region?: Region; gate: ReturnType; guard: ReturnType }; +type Overlay = { vis: ReturnType; catchers: ButtonCatchers; buttons: ButtonRect[]; visible: ReturnType; guide: ReturnType; modelHit?: { x: number; y: number; revision: number; askedAt: number; hit?: boolean }; window: BrowserWindow; display: Display; ready: boolean; queue: SceneMessage[]; pageWantsClicks: boolean; showing: boolean; pointerHeld: boolean; mouseIgnored: boolean; pen: boolean; penOffAt: number; /** The person pressed on the model (E is then ours until they leave it). */ engaged: boolean; /** The model's region on the screen (what the page last said the pointer is over) and the gate that checks the real cursor against it. */ region?: Region; gate: ReturnType; guard: ReturnType }; /** * One overlay window per display: transparent, frameless, always on top, off the taskbar, never @@ -80,6 +81,7 @@ type Overlay = { catchers: ButtonCatchers; buttons: ButtonRect[]; visible: Retur */ export function createOverlayManager(options: OverlayOptions) { const overlays = new Map(); + const alwaysUp = options.backdrop !== undefined || options.size !== undefined; // The E key: taken from the system only while it is wanted (see keyWanted), never by focusing a window. const keys = createKeyHold({ @@ -199,14 +201,23 @@ export function createOverlayManager(options: OverlayOptions) { } }); window.on('page-title-updated', (event) => event.preventDefault()); // keep our own title + // The window is up only while there is something to show (see overlay-visibility.ts); a run with a development backdrop or window size keeps it up as before. window.once('ready-to-show', () => { - if (options.hidden) return; - window.showInactive(); - // Windows places a window by the DPI of the display it started on; set the bounds again once shown. - window.setBounds(display.bounds); + if (alwaysUp) overlay.vis.wake(); }); void window.loadFile(options.page); const overlay: Overlay = { + vis: createOverlayVisibility({ + show: () => { + if (options.hidden || window.isDestroyed()) return; + window.showInactive(); + // Windows places a window by the DPI of the display it started on; set the bounds again once shown. + window.setBounds(overlay.display.bounds); + }, + hide: () => { + if (!window.isDestroyed()) window.hide(); + }, + }), catchers: createButtonCatchers({ makeWindow: () => electronCatcherWindow({ hideFromCapture: options.hideFromCapture }), onPress: (id) => { @@ -250,6 +261,17 @@ export function createOverlayManager(options: OverlayOptions) { return { x: b.x + (r.x as number), y: b.y + (r.y as number), w: r.w as number, h: r.h as number }; } + /** What the window is asked to be for (see overlayOccupied): the page's own list of what it shows, and what the person is in the middle of. */ + function occupied(overlay: Overlay): boolean { + return overlayOccupied({ + alwaysUp, pen: overlay.pen, showing: overlay.showing, holding: overlay.pageWantsClicks || overlay.pointerHeld || overlay.engaged, + busy: options.answerBusy?.() === true, shown: overlay.vis.shown, alphas: overlay.visible.current?.regions.map((r) => r.alpha) ?? [], + }); + } + function settle(overlay: Overlay): void { + if (!overlay.window.isDestroyed()) overlay.vis.update(occupied(overlay)); + } + function find(event: IpcMainEvent): Overlay | undefined { for (const overlay of overlays.values()) { if (overlay.window.webContents === event.sender) return overlay; @@ -279,6 +301,7 @@ export function createOverlayManager(options: OverlayOptions) { } refreshKey(); applyMouse(overlay); + settle(overlay); }); // The user pressed on the model (or its button): E is ours until the pointer leaves it, the click lease runs out or the // pen goes on. The window never takes focus, so the program they are using keeps its keyboard throughout. @@ -296,6 +319,7 @@ export function createOverlayManager(options: OverlayOptions) { escape.showing(overlay.display.id, on); options.onShowing?.(overlay.display.id, on); applyMouse(overlay); + settle(overlay); }); ipcMain.on(CHANNEL.followup, (event, command: unknown) => { const overlay = find(event); @@ -304,7 +328,7 @@ export function createOverlayManager(options: OverlayOptions) { }); ipcMain.on(CHANNEL.visibleRegions, (event, snapshot: unknown) => { const overlay = find(event); - if (overlay && overlay.visible.accept(snapshot)) applyMouse(overlay); + if (overlay && overlay.visible.accept(snapshot)) { applyMouse(overlay); settle(overlay); } }); ipcMain.on(CHANNEL.guideReady, (event, token: unknown, revision: unknown) => { const overlay = find(event), snapshot = overlay?.visible.current; @@ -316,6 +340,7 @@ export function createOverlayManager(options: OverlayOptions) { if (!overlay || typeof on !== 'boolean') return; overlay.pointerHeld = on; applyMouse(overlay); + settle(overlay); }); ipcMain.on(CHANNEL.modelHit, (event, value: unknown) => { const overlay = find(event), pending = overlay?.modelHit; @@ -438,6 +463,9 @@ export function createOverlayManager(options: OverlayOptions) { // A protected guide target must return to the underlying page as the real pointer enters it. const mouseTimer = setInterval(() => { for (const overlay of overlays.values()) if (overlay.showing) applyMouse(overlay); }, 25); mouseTimer.unref(); + // Now and then, whatever else changed (a question ended, a fade finished): look again at whether the window is still needed. + const visibilityTimer = setInterval(() => { for (const overlay of overlays.values()) settle(overlay); }, 250); + visibilityTimer.unref(); function sync(): void { const current = screen.getAllDisplays(); @@ -465,6 +493,7 @@ export function createOverlayManager(options: OverlayOptions) { /** Send a scene message to the overlay of one display; every other display's overlay is cleared. */ send(displayId: number, message: SceneMessage): void { const deliver = (overlay: Overlay, m: SceneMessage) => { + if (wakesOverlay(m)) overlay.vis.wake(); if (m.kind === 'clear' || m.kind === 'dismiss' || ((m.kind === 'begin' || m.kind === 'play') && m.turn <= 1 && !('detail' in m && m.detail))) clearRouting(overlay); let guideToken: number | undefined; if (m.kind === 'status') { @@ -491,8 +520,11 @@ export function createOverlayManager(options: OverlayOptions) { setPen(displayId: number | null): void { for (const [id, overlay] of overlays) { const on = id === displayId; - if (overlay.pen && !on) overlay.penOffAt = Date.now(); + const was = overlay.pen; + if (was && !on) overlay.penOffAt = Date.now(); overlay.pen = on; + // The pen takes the mouse and draws on the screen, and its stroke fades after: the window is up for both. + if (on || was) overlay.vis.wake(); // One window failing must never leave another one holding the mouse. try { if (!overlay.window.isDestroyed()) { @@ -570,6 +602,7 @@ export function createOverlayManager(options: OverlayOptions) { }, dispose(): void { clearInterval(mouseTimer); + clearInterval(visibilityTimer); options.voice?.bridge?.dispose(); options.voice?.system?.dispose(); screen.off('display-added', sync); diff --git a/test/main-overlay-idle.test.ts b/test/main-overlay-idle.test.ts new file mode 100644 index 0000000..1aaf6c7 --- /dev/null +++ b/test/main-overlay-idle.test.ts @@ -0,0 +1,145 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { test } from 'node:test'; +import { HIDE_AFTER_MS, WAKE_MS, createOverlayVisibility, overlayOccupied, wakesOverlay } from '../src/main/overlay-visibility.ts'; + +// The overlay windows cover the whole screen, topmost, over whatever the person watches. They are up only while an answer is asked for, written or shown. + +type Call = [string, ...unknown[]]; + +/** The real overlay manager over a stand-in for Electron (test/support/overlay-idle-child.ts), run in a process of its own so the module can be replaced. */ +function run(mode: string): Call[] { + const child = fileURLToPath(new URL('./support/overlay-idle-child.ts', import.meta.url)); + const r = spawnSync(process.execPath, ['--no-warnings', '--experimental-test-module-mocks', child, mode], { encoding: 'utf8', timeout: 60000 }); + assert.equal(r.status, 0, r.stderr); + return JSON.parse(r.stdout) as Call[]; +} + +/** Whether a window was up at each mark of the log (the marks are named in the child; each is set after what it names). */ +function stateAt(log: Call[], window: string): Record { + const out: Record = {}; + let up = false; + for (const [what, ...rest] of log) { + if (what === `${window}.show`) up = true; + else if (what === `${window}.hide`) up = false; + else if (what === '--') out[rest[0] as string] = up; + } + return out; +} + +test('the overlay window is up for a question and its answer, and down between them', () => { + const log = run('normal'); + const up = stateAt(log, 'window100'); + assert.equal(up['idle at start'], false, 'nothing is up before anyone asks'); + assert.equal(up['a question starts'], false, 'the mark precedes the question'); + assert.equal(up['the answer draws'], true, 'the question raised it and the cue keeps it'); + assert.equal(up['the answer is over, the drawing still up'], true, 'a drawing and a busy answer hold it'); + assert.equal(up['the drawing is gone'], true, 'still the grace'); + assert.equal(up['a pause shorter than the grace'], true, 'a short pause between steps never makes it blink'); + assert.equal(up['past the grace'], false, 'and then it goes'); + assert.equal(up['a clear or dismiss never raises it'], false); + assert.equal(up['busy alone never raises it'], false, 'an answer in hand does not light a window nothing asked for'); + assert.equal(up['a step raises it again'], true); + assert.equal(up['a step that drew nothing lets it go'], false); + assert.equal(up['the pen is on'], true); + assert.equal(up['the pen is off, its stroke fading'], true); + assert.equal(up['the stroke is gone'], false); + assert.equal(up['a guide step waits for the click'], true, 'a guide target keeps the window for as long as it waits'); + assert.equal(up['the guide step is over'], false); + assert.equal(up['a fresh play on display 1 clears display 2 and leaves it down'], true); + assert.equal(stateAt(log, 'window101')['a fresh play on display 1 clears display 2 and leaves it down'], false, 'the other display stays down'); + assert.equal(log.filter(([w]) => w === 'window101.show').length, 0); +}); + +test('the window is up before the page is told, so the first frame is not waiting for it', () => { + const log = run('normal'); + const names = log.map(([w, ...rest]) => (w === 'page100.send' ? `send:${rest[1]}` : w)); + const shown = names.indexOf('window100.show'); + const first = names.indexOf('send:status:hush'); + assert.ok(shown >= 0 && first > shown, 'shown, then bounds set, then the message'); + assert.equal(names[shown + 1], 'window100.bounds', 'the bounds are set once it is up (the display it started on may differ)'); + // every show in the run is followed, before the next message to that page, by its bounds + names.forEach((n, i) => { if (n === 'window100.show') assert.equal(names[i + 1], 'window100.bounds'); }); +}); + +test('a window kept hidden for a development run is never shown, and one with a backdrop to look at stays up', () => { + assert.equal(run('hidden').filter(([w]) => w.endsWith('.show')).length, 0); + const log = run('backdrop'); + assert.equal(log.filter(([w]) => w === 'window100.show').length, 1); + assert.equal(log.filter(([w]) => w === 'window100.hide').length, 0, 'it is not taken down while the backdrop is the point'); +}); + +test('the content protection stays on the window, shown or hidden', () => { + const log = run('normal'); + assert.equal(log.filter(([w]) => w.endsWith('.protect')).length, 2, 'set once per window, when it is made, as before'); +}); + +// ---- the rule itself, with the clock in hand ------------------------------------------------------------------- + +function clock() { + let t = 1000; + const events: string[] = []; + const vis = createOverlayVisibility({ now: () => t, show: () => events.push('show'), hide: () => events.push('hide') }); + return { vis, events, advance: (ms: number) => { t += ms; } }; +} + +test('a wake shows the window once and holds it, even if the page says nothing is drawn yet', () => { + const c = clock(); + assert.equal(c.vis.shown, false); + c.vis.wake(); + c.vis.wake(); + assert.deepEqual(c.events, ['show']); + c.advance(WAKE_MS - 1); + c.vis.update(false); + assert.equal(c.vis.shown, true, 'the page has had its time to say what it draws'); + c.advance(HIDE_AFTER_MS + 1); + c.vis.update(false); + assert.deepEqual(c.events, ['show', 'hide']); +}); + +test('occupied keeps it up, and it goes only after the grace once nothing needs it', () => { + const c = clock(); + c.vis.update(true); + assert.deepEqual(c.events, ['show']); + for (let i = 0; i < 20; i++) { c.advance(1000); c.vis.update(true); } + assert.deepEqual(c.events, ['show']); + c.advance(HIDE_AFTER_MS - 1); + c.vis.update(false); + assert.equal(c.vis.shown, true); + c.advance(1); + c.vis.update(false); + assert.equal(c.vis.shown, false); + c.vis.update(false); + assert.deepEqual(c.events, ['show', 'hide'], 'hidden once, not on every look'); +}); + +test('a message that starts something wakes the window; the ones that end or ask about the pointer do not', () => { + for (const kind of ['play', 'begin', 'step', 'step-open', 'action', 'step-close', 'notice']) assert.equal(wakesOverlay({ kind }), true, kind); + for (const status of ['hush', 'listening', 'working', 'backend', 'detail', 'waiting', 'checking']) assert.equal(wakesOverlay({ kind: 'status', status }), true, status); + assert.equal(wakesOverlay({ kind: 'status', status: 'idle' }), false); + assert.equal(wakesOverlay({ kind: 'pen', on: true }), true); + assert.equal(wakesOverlay({ kind: 'pen', on: false }), false); + for (const kind of ['clear', 'dismiss', 'end', 'press', 'toggle-explode', 'model-hit-query', 'guide-arm', 'screenshot', 'controls', 'seek']) assert.equal(wakesOverlay({ kind }), false, kind); +}); + +test('what the page shows, the pen, a click lease and a busy answer in a window already up keep it; a busy answer alone does not raise one', () => { + const none = { alwaysUp: false, pen: false, showing: false, holding: false, busy: false, shown: false, alphas: [] as number[] }; + assert.equal(overlayOccupied(none), false); + assert.equal(overlayOccupied({ ...none, alphas: [0, 0.01] }), false, 'a faded-out region is not a drawing'); + assert.equal(overlayOccupied({ ...none, alphas: [0.4] }), true); + assert.equal(overlayOccupied({ ...none, pen: true }), true); + assert.equal(overlayOccupied({ ...none, showing: true }), true); + assert.equal(overlayOccupied({ ...none, holding: true }), true); + assert.equal(overlayOccupied({ ...none, busy: true }), false); + assert.equal(overlayOccupied({ ...none, busy: true, shown: true }), true); + assert.equal(overlayOccupied({ ...none, alwaysUp: true }), true); +}); + +test('nothing but the visibility rule shows or hides an overlay window', () => { + const source = readFileSync(new URL('../src/main/overlay-windows.ts', import.meta.url), 'utf8'); + assert.equal((source.match(/\.showInactive\(\)/g) ?? []).length, 1, 'one place shows it'); + assert.equal((source.match(/window\.hide\(\)/g) ?? []).length, 1, 'one place hides it'); + assert.ok(/if \(wakesOverlay\(m\)\) overlay\.vis\.wake\(\);\s+if \(m\.kind === 'clear'/.test(source), 'the window is raised before the page is sent the message'); +}); diff --git a/test/support/overlay-idle-child.ts b/test/support/overlay-idle-child.ts new file mode 100644 index 0000000..2482db5 --- /dev/null +++ b/test/support/overlay-idle-child.ts @@ -0,0 +1,163 @@ +// Run by main-overlay-idle.test.ts in its own process with --experimental-test-module-mocks: the real overlay manager (src/main/overlay-windows.ts) over a +// stand-in for Electron, with the clock under our hand. No window is ever made. It prints what the stand-in saw, as JSON, for the test to judge. + +import { mock } from 'node:test'; + +type Call = [string, ...unknown[]]; +const log: Call[] = []; +const handlers = new Map void>>(); + +class FakeWebContents { + static next = 100; + id = FakeWebContents.next++; + send(...args: unknown[]): void { + const name = (a: unknown): unknown => { + if (!a || typeof a !== 'object' || !('kind' in a)) return a; + const m = a as { kind: string; status?: string }; + return m.status ? `${m.kind}:${m.status}` : m.kind; + }; + log.push([`page${this.id}.send`, ...args.map(name)]); + } + setAudioMuted(): void {} + setWindowOpenHandler(): void {} + on(): void {} + isDestroyed(): boolean { return false; } +} + +class FakeWindow { + static all: FakeWindow[] = []; + webContents = new FakeWebContents(); + bounds: { x: number; y: number; width: number; height: number }; + visible = false; + readonly once$ = new Map void>(); + constructor(options: { x: number; y: number; width: number; height: number }) { + this.bounds = { x: options.x, y: options.y, width: options.width, height: options.height }; + FakeWindow.all.push(this); + } + get tag(): string { return `window${this.webContents.id}`; } + setAlwaysOnTop(): void {} + setIgnoreMouseEvents(): void {} + setContentProtection(on: boolean): void { log.push([`${this.tag}.protect`, on]); } + on(): void {} + once(event: string, fn: () => void): void { this.once$.set(event, fn); } + loadFile(): Promise { return Promise.resolve(); } + showInactive(): void { this.visible = true; log.push([`${this.tag}.show`]); } + hide(): void { this.visible = false; log.push([`${this.tag}.hide`]); } + setBounds(b: { x: number; y: number; width: number; height: number }): void { this.bounds = b; log.push([`${this.tag}.bounds`, b.width, b.height]); } + getBounds(): { x: number; y: number; width: number; height: number } { return this.bounds; } + isDestroyed(): boolean { return false; } + isVisible(): boolean { return this.visible; } + isAlwaysOnTop(): boolean { return true; } + isFocusable(): boolean { return false; } + destroy(): void {} +} + +const displays = [ + { id: 1, bounds: { x: 0, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 }, + { id: 2, bounds: { x: 1920, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 }, +]; + +mock.module('electron', { + namedExports: { + BrowserWindow: FakeWindow, + globalShortcut: { register: () => true, unregister: () => undefined }, + ipcMain: { on: (channel: string, fn: (event: { sender: unknown }, ...args: unknown[]) => void) => { handlers.set(channel, [...(handlers.get(channel) ?? []), fn]); } }, + screen: { getAllDisplays: () => displays, getCursorScreenPoint: () => ({ x: 0, y: 0 }), on: () => undefined, off: () => undefined }, + }, +}); + +mock.timers.enable({ apis: ['Date', 'setInterval', 'setTimeout'] }); + +const { createOverlayManager } = await import('../../src/main/overlay-windows.ts'); +const { CHANNEL } = await import('../../src/shared/ipc.ts'); + +const mode = process.argv[2] ?? 'normal'; +let busy = false; +const manager = createOverlayManager({ + preload: 'preload.js', page: 'overlay.html', hideFromCapture: true, globalKeys: false, + ...(mode === 'hidden' ? { hidden: true } : {}), + ...(mode === 'backdrop' ? { backdrop: 'page.png' } : {}), + answerBusy: () => busy, +}); +manager.start(); + +const emit = (channel: string, window: FakeWindow, ...args: unknown[]): void => { for (const fn of handlers.get(channel) ?? []) fn({ sender: window.webContents }, ...args); }; +const [one, two] = FakeWindow.all as [FakeWindow, FakeWindow]; +for (const w of FakeWindow.all) { w.once$.get('ready-to-show')?.(); emit(CHANNEL.ready, w); } + +let revision = 0; +const snapshot = (regions: Array<{ id: string; kind: string; alpha: number }>) => ({ + version: 1, revision: ++revision, units: 'css-px', width: 1920, height: 1080, + regions: regions.map((r) => ({ ...r, shape: { type: 'rect', x: 10, y: 10, w: 100, h: 40 } })), +}); +const mark = (label: string): void => { log.push(['--', label]); }; +const wait = (ms: number): void => { for (let t = 0; t < ms; t += 250) mock.timers.tick(250); }; + +const steps: Record void> = { + normal() { + mark('idle at start'); + wait(5000); + mark('a question starts'); + manager.send(1, { kind: 'status', status: 'hush' }); + manager.send(1, { kind: 'status', status: 'listening' }); + emit(CHANNEL.visibleRegions, one, snapshot([{ id: 'status', kind: 'pill', alpha: 1 }])); + wait(5000); + mark('the answer draws'); + emit(CHANNEL.visibleRegions, one, snapshot([{ id: 'model-geometry', kind: 'content', alpha: 1 }, { id: 'caption', kind: 'content', alpha: 0.6 }])); + emit(CHANNEL.showing, one, true); + busy = true; + wait(10000); + mark('the answer is over, the drawing still up'); + busy = false; + wait(5000); + mark('the drawing is gone'); + emit(CHANNEL.showing, one, false); + emit(CHANNEL.visibleRegions, one, snapshot([])); + wait(500); + mark('a pause shorter than the grace'); + wait(700); + mark('past the grace'); + manager.send(1, { kind: 'clear' }); + manager.send(1, { kind: 'dismiss' }); + wait(3000); + mark('a clear or dismiss never raises it'); + busy = true; + wait(3000); + mark('busy alone never raises it'); + busy = false; + manager.send(1, { kind: 'step-open', say: 'hello' }); + mark('a step raises it again'); + wait(4000); + mark('a step that drew nothing lets it go'); + manager.setPen(1); + mark('the pen is on'); + wait(2000); + manager.setPen(null); + wait(1000); + mark('the pen is off, its stroke fading'); + wait(4000); + mark('the stroke is gone'); + manager.send(1, { kind: 'status', status: 'waiting' }); + emit(CHANNEL.visibleRegions, one, snapshot([{ id: 'guide-target', kind: 'guide-target', alpha: 1 }, { id: 'status', kind: 'pill', alpha: 1 }])); + wait(30000); + mark('a guide step waits for the click'); + emit(CHANNEL.visibleRegions, one, snapshot([])); + wait(1500); + mark('the guide step is over'); + manager.send(1, { kind: 'play', scene: { v: 1, steps: [] }, turn: 1 } as never); + mark('a fresh play on display 1 clears display 2 and leaves it down'); + wait(0); + }, + hidden() { + manager.send(1, { kind: 'status', status: 'listening' }); + emit(CHANNEL.visibleRegions, one, snapshot([{ id: 'status', kind: 'pill', alpha: 1 }])); + wait(2000); + }, + backdrop() { + wait(10000); + }, +}; +steps[mode]?.(); +manager.dispose(); +void two; +console.log(JSON.stringify(log));