Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/overlay.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ The settings choose what an answer's words are given as (`captions`: `both`, `te
- **Text only.** No voice is asked for and nothing is spoken: the step's whole sentence is shown at once and the step lasts as long as it takes to read it (`readingSeconds`: three words a second and a moment to take the page in, at least 3.5 s), and the drawing is timed to that as it is to a voice's length.
- **Voice only.** No words on screen.

## When the window is on the screen

The overlay is one transparent, topmost window per display, so a window left up all day sits over everything the person watches (a playing video included) whether or not anything is drawn in it. It is therefore hidden while it has nothing to show, and shown without taking focus only while an answer is asked for, written or shown (`src/main/overlay-visibility.ts`). Any message that begins something to see or hear (the cue of a question, the first step, a notice, the pen going on) raises the window in the same moment, before the page is told, and for a few seconds after it whatever the page has said; after that the page's own list of what it shows (the visible regions: the model, marks, diagrams, captions, answer card, pills, a guide target) decides, together with the pen, a held click, and, for a window already up, an answer being asked or written. It goes a moment (under a second) after the last of those, so a pause between steps or a fade never makes it blink. A busy answer never raises a window by itself, so a second display stays down. A development run with a backdrop or a fixed window size keeps the window up as before, and a run that keeps windows hidden never shows them. The page keeps running while the window is down (`backgroundThrottling` is off). The hotkey, the guide, the pins and the click guard do not depend on the window being up: they read the page's regions, which are empty exactly when nothing is shown.

## Fading

When a turn has finished, the scene stays at full strength for `FADE_AFTER_MS` (set in `src/shared/ipc.ts`), then fades over 1.2 s, unless the pointer is on the model or the app's words; touching it starts the count again. A faded scene takes no clicks. It is hidden, not cleared, so a follow-up still reaches its ids; the next question or Clear removes it. `fade.ts` is the logic and is tested without a clock.
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "talkthrough",
"version": "0.7.0",
"version": "0.7.1",
"description": "An AI that talks you through anything on your screen, and draws as it talks.",
"keywords": [
"screen assistant",
Expand Down
91 changes: 91 additions & 0 deletions src/main/overlay-visibility.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// When an overlay window is on the screen. The overlay is a full-screen, transparent, topmost window; left up all day it sits over everything the person
// watches, whether or not anything is drawn in it. So it is hidden while it has nothing to show and shown (without taking focus) only while an answer is
// asked for, written or shown. This holds the rule, with time injected; overlay-windows.ts feeds it what it knows and does the showing.

/** After the last thing that needed the window, how long it stays before it goes (so a pause between steps, or a fade, never makes it blink). */
export const HIDE_AFTER_MS = 800;
/** After a message that starts something (a question, a step, a notice), the window is kept this long even if the page has not yet said what it draws. */
export const WAKE_MS = 3000;

export type OverlayVisibilityDeps = {
show(): void;
hide(): void;
now?: () => number;
hideAfterMs?: number;
wakeMs?: number;
};

export type OverlayVisibility = {
/** Something is about to be drawn or said (a message for the page): the window is up before the page gets it. */
wake(): void;
/** What is known now: whether anything needs the window. Call it when that may have changed, and now and then. */
update(occupied: boolean): void;
readonly shown: boolean;
};

export function createOverlayVisibility(deps: OverlayVisibilityDeps): OverlayVisibility {
const now = deps.now ?? Date.now;
const hideAfter = deps.hideAfterMs ?? HIDE_AFTER_MS;
const wakeFor = deps.wakeMs ?? WAKE_MS;
let shown = false;
/** The last moment the window was known to be needed. */
let needed = -Infinity;
const up = (): void => {
if (shown) return;
shown = true;
deps.show();
};
return {
wake(): void {
needed = Math.max(needed, now() + wakeFor);
up();
},
update(occupied: boolean): void {
const t = now();
if (occupied) {
needed = Math.max(needed, t);
up();
} else if (shown && t - needed >= hideAfter) {
shown = false;
deps.hide();
}
},
get shown(): boolean { return shown; },
};
}

/** The messages to a page that begin something to see or hear. The rest (clear, dismiss, the end of a turn, an idle status, queries about the pointer) only matter when the window is already up. */
export function wakesOverlay(message: { kind: string; status?: string; on?: boolean }): boolean {
switch (message.kind) {
case 'play': case 'begin': case 'step': case 'step-open': case 'action': case 'step-close': case 'notice': return true;
case 'status': return message.status !== 'idle';
case 'pen': return message.on === true;
default: return false;
}
}

export type OccupancyState = {
/** A development run that keeps the window up (a backdrop or a fixed size to look at). */
alwaysUp: boolean;
pen: boolean;
/** The page says a drawing of an answer is up. */
showing: boolean;
/** The page holds a click lease, a press on the model, or the pointer button. */
holding: boolean;
/** An answer is being asked for, written or heard (the session says; it is not tied to one display). */
busy: boolean;
/** This window is already up, so the busy answer is plainly on this display. */
shown: boolean;
/** The alpha of every region the page last published (pills, captions, answer cards, marks, the model). */
alphas: readonly number[];
};

/**
* Whether anything needs the window. The page's own list of what it shows decides what is drawn; the rest keep a window that is up while the person is
* in the middle of something. A busy answer never raises a window by itself (it would light every display): the message that started it does.
*/
export function overlayOccupied(s: OccupancyState): boolean {
if (s.alwaysUp || s.pen || s.showing || s.holding) return true;
if (s.busy && s.shown) return true;
return s.alphas.some((a) => a > 0.02);
}
47 changes: 40 additions & 7 deletions src/main/overlay-windows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import type { ButtonCatchers, ButtonRect } from './button-catchers.ts';
import { electronCatcherWindow } from './button-window.ts';
import { createClickGuard, createGuideArmGate, createKeyGate, createKeyHold, ignoreMouse, keyWanted } from './interaction.ts';
import { createEscape } from './escape.ts';
import { createOverlayVisibility, overlayOccupied, wakesOverlay } from './overlay-visibility.ts';
import type { Region } from './interaction.ts';
import { containsWireContentExceptModel, containsWireId, containsWireRegion, createVisibleRegionReceiver } from './visible-region-wire.ts';
import type { VoiceBridge } from './voice-bridge.ts';
Expand Down Expand Up @@ -71,7 +72,7 @@ export type OverlayOptions = {
onDetail?: (displayId: number) => void;
};

type Overlay = { catchers: ButtonCatchers; buttons: ButtonRect[]; visible: ReturnType<typeof createVisibleRegionReceiver>; guide: ReturnType<typeof createGuideArmGate>; modelHit?: { x: number; y: number; revision: number; askedAt: number; hit?: boolean }; window: BrowserWindow; display: Display; ready: boolean; queue: SceneMessage[]; pageWantsClicks: boolean; showing: boolean; pointerHeld: boolean; mouseIgnored: boolean; pen: boolean; penOffAt: number; /** The person pressed on the model (E is then ours until they leave it). */ engaged: boolean; /** The model's region on the screen (what the page last said the pointer is over) and the gate that checks the real cursor against it. */ region?: Region; gate: ReturnType<typeof createKeyGate>; guard: ReturnType<typeof createClickGuard> };
type Overlay = { vis: ReturnType<typeof createOverlayVisibility>; catchers: ButtonCatchers; buttons: ButtonRect[]; visible: ReturnType<typeof createVisibleRegionReceiver>; guide: ReturnType<typeof createGuideArmGate>; modelHit?: { x: number; y: number; revision: number; askedAt: number; hit?: boolean }; window: BrowserWindow; display: Display; ready: boolean; queue: SceneMessage[]; pageWantsClicks: boolean; showing: boolean; pointerHeld: boolean; mouseIgnored: boolean; pen: boolean; penOffAt: number; /** The person pressed on the model (E is then ours until they leave it). */ engaged: boolean; /** The model's region on the screen (what the page last said the pointer is over) and the gate that checks the real cursor against it. */ region?: Region; gate: ReturnType<typeof createKeyGate>; guard: ReturnType<typeof createClickGuard> };

/**
* One overlay window per display: transparent, frameless, always on top, off the taskbar, never
Expand All @@ -80,6 +81,7 @@ type Overlay = { catchers: ButtonCatchers; buttons: ButtonRect[]; visible: Retur
*/
export function createOverlayManager(options: OverlayOptions) {
const overlays = new Map<number, Overlay>();
const alwaysUp = options.backdrop !== undefined || options.size !== undefined;

// The E key: taken from the system only while it is wanted (see keyWanted), never by focusing a window.
const keys = createKeyHold({
Expand Down Expand Up @@ -199,14 +201,23 @@ export function createOverlayManager(options: OverlayOptions) {
}
});
window.on('page-title-updated', (event) => event.preventDefault()); // keep our own title
// The window is up only while there is something to show (see overlay-visibility.ts); a run with a development backdrop or window size keeps it up as before.
window.once('ready-to-show', () => {
if (options.hidden) return;
window.showInactive();
// Windows places a window by the DPI of the display it started on; set the bounds again once shown.
window.setBounds(display.bounds);
if (alwaysUp) overlay.vis.wake();
});
void window.loadFile(options.page);
const overlay: Overlay = {
vis: createOverlayVisibility({
show: () => {
if (options.hidden || window.isDestroyed()) return;
window.showInactive();
// Windows places a window by the DPI of the display it started on; set the bounds again once shown.
window.setBounds(overlay.display.bounds);
},
hide: () => {
if (!window.isDestroyed()) window.hide();
},
}),
catchers: createButtonCatchers({
makeWindow: () => electronCatcherWindow({ hideFromCapture: options.hideFromCapture }),
onPress: (id) => {
Expand Down Expand Up @@ -250,6 +261,17 @@ export function createOverlayManager(options: OverlayOptions) {
return { x: b.x + (r.x as number), y: b.y + (r.y as number), w: r.w as number, h: r.h as number };
}

/** What the window is asked to be for (see overlayOccupied): the page's own list of what it shows, and what the person is in the middle of. */
function occupied(overlay: Overlay): boolean {
return overlayOccupied({
alwaysUp, pen: overlay.pen, showing: overlay.showing, holding: overlay.pageWantsClicks || overlay.pointerHeld || overlay.engaged,
busy: options.answerBusy?.() === true, shown: overlay.vis.shown, alphas: overlay.visible.current?.regions.map((r) => r.alpha) ?? [],
});
}
function settle(overlay: Overlay): void {
if (!overlay.window.isDestroyed()) overlay.vis.update(occupied(overlay));
}

function find(event: IpcMainEvent): Overlay | undefined {
for (const overlay of overlays.values()) {
if (overlay.window.webContents === event.sender) return overlay;
Expand Down Expand Up @@ -279,6 +301,7 @@ export function createOverlayManager(options: OverlayOptions) {
}
refreshKey();
applyMouse(overlay);
settle(overlay);
});
// The user pressed on the model (or its button): E is ours until the pointer leaves it, the click lease runs out or the
// pen goes on. The window never takes focus, so the program they are using keeps its keyboard throughout.
Expand All @@ -296,6 +319,7 @@ export function createOverlayManager(options: OverlayOptions) {
escape.showing(overlay.display.id, on);
options.onShowing?.(overlay.display.id, on);
applyMouse(overlay);
settle(overlay);
});
ipcMain.on(CHANNEL.followup, (event, command: unknown) => {
const overlay = find(event);
Expand All @@ -304,7 +328,7 @@ export function createOverlayManager(options: OverlayOptions) {
});
ipcMain.on(CHANNEL.visibleRegions, (event, snapshot: unknown) => {
const overlay = find(event);
if (overlay && overlay.visible.accept(snapshot)) applyMouse(overlay);
if (overlay && overlay.visible.accept(snapshot)) { applyMouse(overlay); settle(overlay); }
});
ipcMain.on(CHANNEL.guideReady, (event, token: unknown, revision: unknown) => {
const overlay = find(event), snapshot = overlay?.visible.current;
Expand All @@ -316,6 +340,7 @@ export function createOverlayManager(options: OverlayOptions) {
if (!overlay || typeof on !== 'boolean') return;
overlay.pointerHeld = on;
applyMouse(overlay);
settle(overlay);
});
ipcMain.on(CHANNEL.modelHit, (event, value: unknown) => {
const overlay = find(event), pending = overlay?.modelHit;
Expand Down Expand Up @@ -438,6 +463,9 @@ export function createOverlayManager(options: OverlayOptions) {
// A protected guide target must return to the underlying page as the real pointer enters it.
const mouseTimer = setInterval(() => { for (const overlay of overlays.values()) if (overlay.showing) applyMouse(overlay); }, 25);
mouseTimer.unref();
// Now and then, whatever else changed (a question ended, a fade finished): look again at whether the window is still needed.
const visibilityTimer = setInterval(() => { for (const overlay of overlays.values()) settle(overlay); }, 250);
visibilityTimer.unref();

function sync(): void {
const current = screen.getAllDisplays();
Expand Down Expand Up @@ -465,6 +493,7 @@ export function createOverlayManager(options: OverlayOptions) {
/** Send a scene message to the overlay of one display; every other display's overlay is cleared. */
send(displayId: number, message: SceneMessage): void {
const deliver = (overlay: Overlay, m: SceneMessage) => {
if (wakesOverlay(m)) overlay.vis.wake();
if (m.kind === 'clear' || m.kind === 'dismiss' || ((m.kind === 'begin' || m.kind === 'play') && m.turn <= 1 && !('detail' in m && m.detail))) clearRouting(overlay);
let guideToken: number | undefined;
if (m.kind === 'status') {
Expand All @@ -491,8 +520,11 @@ export function createOverlayManager(options: OverlayOptions) {
setPen(displayId: number | null): void {
for (const [id, overlay] of overlays) {
const on = id === displayId;
if (overlay.pen && !on) overlay.penOffAt = Date.now();
const was = overlay.pen;
if (was && !on) overlay.penOffAt = Date.now();
overlay.pen = on;
// The pen takes the mouse and draws on the screen, and its stroke fades after: the window is up for both.
if (on || was) overlay.vis.wake();
// One window failing must never leave another one holding the mouse.
try {
if (!overlay.window.isDestroyed()) {
Expand Down Expand Up @@ -570,6 +602,7 @@ export function createOverlayManager(options: OverlayOptions) {
},
dispose(): void {
clearInterval(mouseTimer);
clearInterval(visibilityTimer);
options.voice?.bridge?.dispose();
options.voice?.system?.dispose();
screen.off('display-added', sync);
Expand Down
Loading
Loading