diff --git a/.dockerignore b/.dockerignore index d31d0915..7dd9be9e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -29,3 +29,6 @@ docs/screenshots # that survives every `docker save` and every registry push. stem_key *.tar + +# The relay is a separate Go program with its own image (relay/Dockerfile). +relay diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25313ec9..1f00a901 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,12 @@ jobs: - name: Headless server boots under plain node run: npm run test:server + # tests/e2e/browser-live.spec.ts drives a real Chromium with the Stem + # extension through the native host — the one check of the per-platform + # browser-control glue. It skips itself when no browser is installed. + - name: Install Chromium for the browser-control E2E + run: npx playwright install ${{ runner.os == 'Linux' && '--with-deps' || '' }} chromium + - name: E2E tests (Playwright + Electron) run: ${{ runner.os == 'Linux' && 'xvfb-run --auto-servernum --' || '' }} npm run test:e2e @@ -94,3 +100,17 @@ jobs: - name: Unit tests (Vitest, Windows-relevant subset) run: npm run test:win + + relay: + runs-on: ubuntu-latest + defaults: + run: + working-directory: relay + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: relay/go.mod + cache-dependency-path: relay/go.sum + - run: go vet ./... + - run: go test ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 800bb564..33470665 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,10 @@ name: Release on: push: tags: ['v*'] + # Run by hand (Actions → Release → Run workflow, or `gh workflow run release.yml + # --ref `) to build every installer from a branch before tagging; the + # installers land as workflow artifacts and nothing is published. + workflow_dispatch: permissions: contents: write @@ -121,13 +125,30 @@ jobs: env: APPLE_API_KEY_P8: ${{ secrets.APPLE_API_KEY_P8 }} # Sign while electron-builder's temporary signing keychain is available. - - run: npm run dist:mac -- --arm64 --config.forceCodeSigning=true --config.dmg.sign=true + # Stapling fetches the ticket from Apple's CloudKit, and the connection + # from GitHub's macOS runners drops now and then (Error 68, "The network + # connection was lost") after Apple has already accepted the app. Build + # again rather than fail the release on it. + - name: Build, sign and notarize + run: | + for attempt in 1 2 3; do + if npm run dist:mac -- --arm64 --config.forceCodeSigning=true --config.dmg.sign=true; then exit 0; fi + echo "::warning::dist:mac failed (attempt $attempt)" + rm -rf release + sleep 30 + done + exit 1 - name: Verify signed and notarized app run: | app=release/mac-arm64/Stem.app test "$(lipo -archs "$app/Contents/MacOS/Stem")" = arm64 codesign --verify --deep --strict --verbose=2 "$app" - xcrun stapler validate "$app" + # validate asks Apple's CloudKit too; see the build step above. + for attempt in 1 2 3 4 5 6; do + if xcrun stapler validate "$app"; then break; fi + if [ "$attempt" = 6 ]; then exit 1; fi + sleep 20 + done spctl --assess --type execute --verbose=4 "$app" - name: Notarize and verify final DMG run: bash scripts/notarize-mac-dmg.sh release/*-arm64.dmg @@ -152,8 +173,41 @@ jobs: if: always() run: rm -f "$RUNNER_TEMP/AuthKey.p8" + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: npm + - run: npm ci + # Unsigned: no Windows code-signing certificate yet (see the `win` block + # in electron-builder.yml). + - run: npm run dist:win + - name: Packaged app is sound + shell: bash + run: | + app=release/win-unpacked/resources/app + node "$app/node_modules/@earendil-works/pi-coding-agent/dist/cli.js" --version + for stray in src tests ios docs scripts Dockerfile; do + if [ -e "$app/$stray" ]; then echo "::error::$stray/ leaked into the packaged app"; exit 1; fi + done + ls "$app/node_modules/onnxruntime-node/bin/napi-v6" + # latest.yml + the installer's blockmap are what an installed Stem + # updates from (src/desktop/updates.ts). + - uses: actions/upload-artifact@v4 + with: + name: windows + if-no-files-found: error + path: | + release/Stem-Setup-*.exe + release/Stem-Setup-*.exe.blockmap + release/latest.yml + publish: - needs: [linux-x64, mac] + needs: [linux-x64, linux-arm64, mac, windows] + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v4 diff --git a/.gitignore b/.gitignore index 0babebfe..04ebfb03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -node_modules/ +node_modules dist/ out/ # electron-builder artifacts (see electron-builder.yml directories.output) diff --git a/README.md b/README.md index 6e1de918..64aee1dc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

Download - · macOS (Apple Silicon) · Linux (AppImage, deb) · + · macOS (Apple Silicon) · Linux (AppImage, deb) · Windows · User guide

@@ -114,7 +114,7 @@ A global shortcut opens Quick Chat over whatever app you're in. Ask, hit Enter, - **Coding agents.** A persona hands work to Claude Code or any other ACP agent, on the server or on one of your computers. The agent's permission requests arrive as cards in the chat. - **Computer control (alpha).** A persona pinned to your Mac can work in its apps, including windows on another Space or behind other windows. macOS 14 or newer. - **Skill recorder (alpha).** Record yourself doing a task on the Mac. Stem writes it up as a draft skill, and a practice run tries it before you save it. -- **Browser control.** The Stem extension for Chrome, Arc, Dia, Brave, Edge and Chromium lets Stem use your own signed-in browser in background tabs. +- **Browser control.** The Stem extension for Chrome, Arc, Dia, Brave, Edge and Chromium lets Stem use your own signed-in browser in background tabs, on a Mac or Linux. - **Scheduled tasks.** Ask for "every weekday at 8, check…" and you get a cron job. Each run starts in a fresh thread. - **Shell commands.** A safety check runs what you asked for and turns the rest back. You decide only after repeated refusals, and you can always allow commands by name or pattern. - **MCP servers and skills.** Add MCP servers by command or URL, OAuth included. Stem writes skills from work that went well and rewrites the ones that failed. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index aa5c9a12..35aa1cca 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -12,6 +12,77 @@ Maintainer notes: date, tag. --> +## 0.7.0 — 2026-10-19 + +### Added + +- **Mail with other people's Stems.** Add your email address under Settings → Features → **Your + email address** and confirm the code Stem emails you. People who use Stem can then mail you there, + and you can mail them. Mail is encrypted from one Stem to the other and is text only for now. Mail + from someone new waits under **Requests** until you accept them. Your personas see an incoming + mail only after you let them, and a reply a persona drafts goes out only after you approve it. + **Only people I've written to can reach me** keeps everyone else from finding out that you use + Stem. Compare keys in person to mark a contact as checked. Requests and drafts are on the desktop + only. +- **Share links.** Hover an answer and click the share icon to publish the whole chat, the question + and answer, or the answer alone as a link anyone can open. The content is encrypted in your Stem, + and the key is part of the link, so the server that hosts it can't read it. Links show a preview + card in chat apps, stay out of search engines, and expire after 30 days. Settings lists your links + so you can take one down. Sharing needs a verified email address, and a company that limits mail + with other Stems turns it off. It works from the iPhone too. +- **Ask about this window.** A second Quick Chat shortcut opens Quick Chat next to the window you're + in, with a picture and the text of that window attached. Ask "What does Jana need from me?" in + Slack and the answer appears beside it. If one of your MCP servers covers that app, Stem uses it + to read what the window doesn't show. With computer control allowed, Stem can also click and type + in that window. Set the shortcut under Settings → Quick Chat. Works on the Mac, Windows and Linux + (GNOME). +- **Computer control on Windows and Linux (Alpha).** "Let Stem control this computer" now works on + Windows 10 and 11, and on Linux under GNOME on Wayland. On Linux, Stem installs a small GNOME + extension from Settings → Features → Computer control. You log out and back in once after the + first install, and later updates arrive without a login. Moving your own mouse or pressing a key + stops a run. +- **Browser control on Windows and Linux (Beta).** The Stem extension now works in Chrome, Edge, + Brave and Chromium on Windows and Linux, and in Chromium-based AppImages such as Helium. Browsers + installed as Snap or Flatpak can't run it. +- **Several browser profiles.** If you keep separate browser profiles for work and home, each one + connects on its own. Name them under Settings → Features → Browser control, pick the one tasks + start in, and limit a persona or chat to the profiles it may use. Asked for your open tabs, Stem + lists them by profile. +- **Dictation on Windows and Linux (Beta).** The microphone button now works there too, with Whisper + running on your computer. Its model downloads the first time you use it, and the audio never + leaves your computer. +- **Secretary brings in help on its own.** A Secretary mail now adds whichever personas the job + needs, including ones that control your computer, and sees the errand through. Use **Normal** if + you want to add personas yourself. + +### Changed + +- **The iPhone opens faster.** The chat list and mail show the last list at once and load less over + the network, so a slow connection no longer means a long wait. +- **Connected folders skip ignored files.** A mirrored folder leaves out what its `.gitignore` + leaves out. + +### Fixed + +- **Several pictures made at once all show.** When Stem made a few pictures in one go, all but the + last read "no longer available" until you reopened the chat. +- **A long chat on the iPhone no longer goes blank** after you send a message. +- **Every HEIC photo from the iPhone arrives.** Some HEIC and HEIF variants were sent without being + converted to JPEG and couldn't be read. +- **A scheduled task survives a server restart.** A run the server died under is now recorded and + run once more, instead of being lost. +- **Long sign-ins finish.** Signing in during onboarding no longer gets stuck when it takes more + than five minutes. +- **Answered approvals leave the Inbox.** Answering an approval in mail moves that conversation out + of the Inbox, and archiving a chat clears its Return-to-chat row. +- **Named chats stay read.** A chat Stem named after its first turn no longer turned bold again. +- **The working dots stay** under a reply that's still being written, so you can tell it isn't + finished. +- **Big servers start.** On a server with many chats, the chat list scan at startup could run out of + memory. +- **The first launch on Windows works.** Windows scans every file on the first launch after an + install or update, which made Stem time out before it could answer. + ## 0.6.1 — 2026-10-13 ### Fixed diff --git a/deploy/env.example b/deploy/env.example index 638163f8..3fdb8226 100644 --- a/deploy/env.example +++ b/deploy/env.example @@ -53,10 +53,15 @@ TZ=Europe/Bratislava # ---- Push notifications to the iOS app (optional; off unless all four are set) ---- # -# Without these, Stem sends no notifications and does no work towards them — the -# server is exactly what it was. With them, an approval card, a long turn ending -# or a task alert also taps the phone on the shoulder; the notification carries -# ids and a short label and never a message (src/server/push/). +# You don't need these. Without them, Stem wakes the phone through the global +# relay's push gateway, which shows fixed wording only ("Approval needed") and +# sees the device token and ids, never a message. STEM_PUSH_GATEWAY=off turns +# that off; STEM_PUSH_GATEWAY_URL points it at another relay. +# +# With them, this server talks to Apple itself: an approval card, a long turn +# ending or a task alert taps the phone on the shoulder with ids and a short +# label and never a message (src/server/push/). Only the holder of the app's +# APNs key can do this. # # The key is the .p8 Apple hands you once, for a key with the APNs service # enabled. It has to be INSIDE the container, so mount it in docker-compose.yml diff --git a/docs/README.md b/docs/README.md index f9e1a3a8..51be7c03 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,8 @@ Pick the feature you see in Stem. | [Chats](user/chats.md) | Ask, attach files, organize conversations | | [Quick Chat](user/quick-chat.md) | Ask from any app | | [Mail](user/mail.md) | Send requests to personas and get replies in your Inbox | +| [Mail with other Stems](user/other-stems.md) | Write to people who use Stem, with your personas drafting the replies | +| [Share links](user/share-links.md) | Publish an answer or a chat as a link anyone can open | | [Memory](user/memory/README.md) | Learn useful context and recall earlier chats | | └ [Facts](user/memory/facts.md) | Durable details: preferences, people, projects | | └ [Past chats](user/memory/recall.md) | Recall: find relevant parts of past conversations | diff --git a/docs/assistant/assistant-host.md b/docs/assistant/assistant-host.md index d317be64..dd0d5711 100644 --- a/docs/assistant/assistant-host.md +++ b/docs/assistant/assistant-host.md @@ -2,6 +2,6 @@ Stem is NOT running on the computer the user is typing on. Stem itself runs on a server they own (Linux, macOS or Windows); the app in front of them is a client talking to it over the network, and their phone may be another. Your `run_command` shell, the files you read and write, and every MCP server that is not pinned to one of their computers run ON THAT SERVER. -So: the server has what a server has installed, not what their own computer has, and it cannot reach an address on their home network. When something fails there because a program is missing, say plainly which machine is missing it instead of asking them to check their own. Do not tell them your shell "does not run on your computer" as though you had no shell — you have the server's, which is the right one for anything belonging to Stem. If a job genuinely has to happen on their own computer, you may have two ways in: `run_command` with its `device` parameter runs a shell command on one of their own computers (only one that has switched on "accepts commands" — your per-turn context names them), and an MCP server pinned to that computer covers everything else; read the `assistant-mcp` and `mcp-servers` guide pages. The `device` shell is for shell work — files, git, scripts, `open -a`. Driving that computer's screen (System Settings, clicking, typing, AppleScript at System Events or at an app) is the job of the `computer` tool: a persona pinned to that computer in the persona editor has it, and so does a chat with no persona when the user allowed computer control in chats (Settings → Features). A persona chat without a pin never has it, whatever that switch says. When a persona is pinned to the computer, a GUI-scripting command from a conversation that may not drive it is refused with the persona's name. Hand the task to that persona (send_mail it in a mail conversation the user added it to) or tell the user to ask it, rather than scripting around it. Coding agents follow the same rule: a persona's coding setup, or Settings → Features → Coding agents for chats with no persona. So does the user's browser: the `browser` tool (the Stem extension in their own Arc or Chrome) comes from a persona's browser pin or Settings → Features → Browser control; web pages go through it, not through `computer` or a browser opened over `run_command`. +So: the server has what a server has installed, not what their own computer has, and it cannot reach an address on their home network. When something fails there because a program is missing, say plainly which machine is missing it instead of asking them to check their own. Do not tell them your shell "does not run on your computer" as though you had no shell — you have the server's, which is the right one for anything belonging to Stem. If a job genuinely has to happen on their own computer, you may have two ways in: `run_command` with its `device` parameter runs a shell command on one of their own computers (only one that has switched on "accepts commands" — your per-turn context names them), and an MCP server pinned to that computer covers everything else; read the `assistant-mcp` and `mcp-servers` guide pages. The `device` shell is for shell work — files, git, scripts, `open -a`. Driving that computer's screen (System Settings, clicking, typing, AppleScript at System Events or at an app) is the job of the `computer` tool, on a Mac, on Windows 10 and 11, and on Linux under GNOME on Wayland: a persona pinned to that computer in the persona editor has it, and so does a chat with no persona when the user allowed computer control in chats (Settings → Features). A persona chat without a pin never has it, whatever that switch says. When a persona is pinned to the computer, a GUI-scripting command from a conversation that may not drive it is refused with the persona's name. Hand the task to that persona (send_mail it in a mail conversation the user added it to) or tell the user to ask it, rather than scripting around it. Coding agents follow the same rule: a persona's coding setup, or Settings → Features → Coding agents for chats with no persona. So does the user's browser: the `browser` tool (the Stem extension in their own Chrome-family browser: Arc, Chrome, Edge, Brave and the like) comes from a persona's browser pin or Settings → Features → Browser control; web pages go through it, not through `computer` or a browser opened over `run_command`. A program the server lacks is often still one command away, so try these before declaring it missing: anything published on PyPI runs as `uvx ` (`uvx yt-dlp `) and anything on npm as `npx -y `, with no install step — the download is kept, so it is only slow the first time. A tool you will reach for again can go on the PATH with `uv tool install `; those land in a folder that survives server upgrades. `apt-get install` also works (the run pauses for the user's approval) and is the way to a system-level program like `ffmpeg` — but an upgrade replaces the container and an apt install goes with it, so when you resort to apt, say so and tell the user that a few lines in a `Dockerfile.local` beside the server's docker-compose.yml make it permanent (the "Running on a server" doc in the Stem repository shows exactly the lines). diff --git a/docs/browser-control.md b/docs/browser-control.md index e4323de0..e8ad26b5 100644 --- a/docs/browser-control.md +++ b/docs/browser-control.md @@ -25,9 +25,10 @@ The user chose the computer-control posture instead. | Actions | `tabs`, `open`, `navigate`, `snapshot` (an accessibility outline with refs), `screenshot`, `click`, `hover`, `type`, `fill`, `press`, `scroll`, `wait`, `dialog`, `evaluate`, `console`, `network`, `upload`, `downloads`, `close`. | | Uploads | Only files Stem holds: Files, scratch, connected folders, `img_` ids, and the conversation's attachments, which a browser turn keeps as files. Never an arbitrary path on the Mac. | | Downloads | Copied into the conversation's scratch `downloads/` on the server. | -| Several browsers | One per Mac, chosen in that Mac's Settings. The chosen browser is started (`open -g -a`) when a run needs it and it is closed. | +| Several browsers | One per computer, chosen in that computer's Settings. The chosen browser is started when a run needs it and it is closed: `open -g -a` on a Mac; on Linux its executable is run unless a process already runs it, and it may come to the front. | +| Several profiles (2026-10-13) | Every profile with the extension connects on its own and stays connected. Settings names them and picks the default; the model names another with `profile`, and that profile sticks to the thread until the run ends. `tabs` with no profile lists every open profile's tabs. See below. | | Concurrency | Several runs at once, each keeping track of its own tabs, with one action queue per tab. A run is warned when another run used the same tab recently. | -| Platform | macOS only for now: Arc, Chrome, Dia, Brave, plus Chromium, Edge and Chrome for Testing when they are installed. | +| Platform | macOS: Arc, Chrome, Dia, Brave, plus Chromium, Edge and Chrome for Testing when they are installed. Linux: Chrome, Brave, Chromium, Edge, plus every other Chromium user-data folder under `$XDG_CONFIG_HOME` (one or two levels down, `Local State` + `Default/`, which leaves Electron apps out) — Helium, Thorium, Vivaldi, Opera, beta channels. The browser is named by `/proc//exe`; for an AppImage (exe inside the parent's `APPDIR`) by its `APPIMAGE` file instead, since the mount changes every launch. Stem starts known system binaries, and AppImages that are a regular executable file with the AppImage magic, owned by the user or root and writable by no one else; a tarball browser in the home folder connects but is never started. Snap and Flatpak browsers are out: their sandbox reads its own manifest folder and will not start a host outside it. Windows: no. | | Install | An unpacked extension: Set up copies it into the profile's state folder and writes native-messaging manifests. The extension id is fixed by the manifest `key`. A Web Store listing comes later. | | Local server | Works when the server runs on the same Mac. Computer control was opened up to that case at the same time. | @@ -56,6 +57,37 @@ pi `browser` tool ──ctx.ui.input(stem-browser-bridge)──▶ PiRuntime.han - The native host keeps retrying the socket while Stem is down. The open native port keeps the extension's service worker alive, so it sees Stem come back. +## Profiles + +Each browser profile runs its own copy of the extension, and so its own native host. +Until 0.7.0 the desktop keyed connections by the app bundle the host was launched from, +so Arc's work and personal profiles replaced each other's connection every few seconds. +Now the extension keeps a random id in `chrome.storage.local` (one per profile) and sends +it in its `hello`, along with the three sites most open there. The desktop keys +connections by `#`, and a connection counts only once that hello +has arrived (the native host replays it on every socket connection). + +- Names: the user's label from Settings, otherwise "Arc 2" by connect order when an app + has several profiles, otherwise the app's name. The desktop sends each extension its + name (`{type: 'profile'}`) for the toolbar popup, which is how the user tells the + profiles apart. The sites hint is shown only in that Mac's Settings and never goes to + the server. +- Routing: `DeviceBrowserRequest.profile` is the name the model passed. The desktop + resolves it (a profile's name, or an app with one profile), and it then sticks to the + thread until the `end` frame. With no name, the thread's last profile is used, or the + default. +- `tabs` without a profile on a Mac with several asks every connected profile, plus the + thread's own one (started if needed), and answers with one section per profile. Arc's + sidebar completion is added once. +- Entries remembered before profiles (the bare app path) are taken over by the first + profile of that app to connect, keeping their name and their place as the default. +- An extension older than 0.7.0 sends no id and is keyed by the app path, as before. +- Starting a closed profile: Stem can only start the app (`open -g -a`). Chrome then + opens its last-used profile. When the wanted profile does not connect, the error tells + the model to ask the user to open a window of it. +- A persona pin to a single profile is not built. The model picks the profile from the + task. + ## Arc's unloaded tabs Arc gives extensions only the tabs it has loaded since it started. A sidebar tab diff --git a/docs/inter-stem-mail-design.md b/docs/inter-stem-mail-design.md new file mode 100644 index 00000000..421b4dd4 --- /dev/null +++ b/docs/inter-stem-mail-design.md @@ -0,0 +1,523 @@ +# Inter-Stem mail design + +Status: decided 2026-10-13, revised the same day after review (discoverability, +sender authentication) and after the Stem chat "Stem Inter-Stem Mail Design" +(purpose, full memory by default). Nothing built. Builds on the mail system +(`src/server/mail/router.ts`, `MailItem` in `src/shared/types.ts`) and the +Inbox, which was groundwork for this from 0.4.0. + +## Goal + +Not another email. Email and messaging apps already carry words between +people. What Stem adds is that **each side has an assistant that already +knows its owner's situation**, so two people can work something out without +either one packaging all the context or relaying every clarification: + +- **Making plans together.** "Find a weekend activity for our families." + Each Stem knows its own family's constraints and proposes what fits, + without handing the other side its owner's personal context. +- **Delegating across people.** A colleague asks for a review of an + infrastructure change. Their Stem brings the proposal and background; + yours investigates, asks for what's missing, and returns your conclusions. +- **Keeping a shared commitment moving.** "We agreed to record a podcast + next month." Each Stem tracks its owner's part, gathers material and flags + blockers. + +The model is **humans let in, agents prepare, humans release.** Your agents use your full +memory and tools to prepare each reply; you review the exact outgoing mail +and send it. The value isn't fewer clicks, it's no work behind each click: +gathering context, investigating, comparing options, remembering what's +still open. + +A remote conversation is therefore shown as an **ongoing collaboration with +a goal and pending decisions** ("Podcast recording: waiting for Martin's +availability"), not just a thread of messages. + +Mail between Stems uses the email addresses people already have, reaches the +person rather than their agents, and is end-to-end encrypted. + +## Settled decisions + +1. **One person per Stem.** No shared multi-user servers. Most Stems run + embedded in the desktop app on a laptop, so they are often asleep or + offline. A VPS-hosted Stem is the same thing, just always on. +2. **Addresses are real email addresses** (`martin.mrkvicka@gmail.com`), not + relay handles. Owning the address is proven by an emailed code or link. +3. **Relays.** Awantech runs the global relay. A company can run a private + relay for its own domain. Both run the same software. +4. **End-to-end encrypted.** A relay sees who mails whom, when, and how big + the mail is. It never sees subject, body or attachments. +5. **Every incoming mail is approved by the human before any agent sees + it**, including mail from accepted contacts and mid-collaboration + replies. An incoming mail never starts a persona turn on its own. +6. **Every outbound persona mail to another Stem is approved by the human**, + whether it is a reply the human asked for or a mail the persona started. +7. **The relay only carries mail in transit.** It deletes a mail once the + recipient's Stem confirms it has it, or when the wait runs out. It is not a + mailbox. +8. **Losing the laptop loses everything**, the key included, the same as the + rest of Stem. There is no key backup. Recovery is setting up again and + verifying the address again. +9. **Your agents use your full memory in external conversations, by + default.** Nothing is shared with the other Stem except what you approve + in an outgoing mail. The other side's agents never get access to your + memory. +10. **Membership is not hidden** from people who know an address, unless the + owner turns on Unlisted. +11. **A sender's address is vouched for by its home relay** with short-lived + certificates; DNS is the trust root, as for email. +12. **Every outgoing mail is approved, in full**, for now. Bounded + collaborations where agents handle routine exchanges themselves are a + later step. +13. **Exits other than mail are enforced in code** in external conversations + (tool set), since approval only covers mail. + +## Identity and keys + +- Each Stem has one keypair: X25519 for encryption, Ed25519 for signing. The + private key never leaves the Stem. +- A Stem can register several addresses (work and personal) with the same + key. +- Registration: Stem sends `{address, publicKey}` to the relay. The relay + emails a code or link to the address. Confirming it publishes the mapping + address → key. +- The relay answers every registration with "code sent", whether the address + is new or already registered. If it is registered, the email tells the + owner someone tried to add a key and to ignore it unless it was them. +- A new key (new laptop) goes through the same email verification. Contacts + see a key-change notice in the conversation. + +### The relay vouches for keys + +Because the relay maps addresses to keys, whoever runs the relay could hand +out a false key and read mail. The design limits this: + +- **Pinning.** A Stem pins a contact's key on first contact. A changed + key is shown loudly in the conversation, and nothing is sent to the new + key until the user acknowledges the change. +- **In-person verification.** Comparing a fingerprint or scanning a QR code + upgrades a contact from "verified by email" to "verified by you". +- **Key log (later).** An append-only public log of address → key mappings, + which Stems check, so a false key handed to one person is detectable. This + is what makes "the relay can't read your mail" true even about the operator. + +## Relays and routing + +- Which relay serves an address comes from its domain. A TXT record at + `_stem.` names the domain's relay. Without a record, the address + belongs to the global relay (gmail.com and everything else). +- The global relay refuses to register addresses in any domain that has its + own record, so nobody can claim `martin@acme.com` before Acme sets up. +- A private relay verifies its people however it likes: email codes first, + SSO later. +- Stems connect outward only: each Stem holds a long-lived connection to the + relay of each of its addresses. No open ports, works behind NAT and company + firewalls. +- Sending to an address on another relay goes to that relay directly. Relays + don't forward to each other. +- **A company can keep Stem mail inside.** Three locks, each usable alone: + - a **policy file** on company machines (`docs/stem-policy.md`) turns + inter-Stem mail off, or allows only the company's domains through the + company relay. The app then never contacts the global relay (push + gateway included), and the user can't loosen it from Settings; + - a **closed relay** (`closed = true`) refuses every sender outside its + domains and named partners, and answers their lookups as "not on Stem"; + - `access=closed` in the `_stem` record makes every other relay, the + global one first, refuse the domain's people as senders, which also + covers employees whose Stem has no policy file. + +### Delivery + +1. The sender's Stem looks up the recipient's key (cached and pinned after + the first time), seals the mail to it, signs it, attaches its sender + certificate (see Sender authentication), and posts it to the + recipient's relay. +2. The relay checks the certificate, then holds the sealed mail until the + recipient's Stem collects it and confirms it has stored it, then deletes + it. The wait is capped at 7 days; an expired mail is deleted, and between + accepted contacts the sender's Stem marks it not delivered. +3. Approved sends made while offline queue on the sender's Stem and go out + when it reconnects. +4. A lookup that finds no key means "not reachable": the address isn't on + Stem, or its owner is unlisted (see Discoverability). The sender's Stem + can't tell which. It offers "invite them to Stem"; the relay emails an + invite only if the address is unregistered and the invite limits allow, + and never reports whether it did. The mail is kept on the sender's Stem + and offered for sending once that address becomes reachable. + +The global relay needs a transactional mail provider (Postmark or SES) for +verification and invite emails; a bare VPS lands in spam. + +### What the relay knows + +Registered addresses and keys, and who mailed whom, when, and the size. Under +real email addresses this is a social graph. The privacy note says so. + +## Sender authentication + +An envelope signature proves the sender has a key, not that they own the +address in `from`. The sender's **home relay** (the relay that verified the +address) vouches for that, the same way DKIM lets a domain vouch for mail: + +- **Relay signing keys.** Every relay has a long-lived signing key. A private + relay publishes it in its `_stem.` TXT record (alongside the relay + URL). The global relay's key ships inside Stem and is also served at + `/.well-known/stem-relay`. DNS is the trust root for a domain, as it is for + email; no relay vouches for addresses outside its domains. +- **Sender certificates.** The home relay issues each verified Stem a + certificate per address: `{address, key fingerprint, issuing relay, + expires}` signed by the relay's key, valid 24 hours and refreshed + automatically while the Stem is connected. +- **On send,** the envelope carries the certificate and is signed with the + key it names. The receiving relay checks: the certificate's issuer is the + authority for the address's domain (DNS record, or the global relay when + there is none), the issuer signature, expiry, and the envelope signature + against the certified key. Anything that fails is refused. +- **The recipient's Stem checks the same things itself**, and doesn't trust + its own relay's word. It also compares the certified key with the pinned + one, so a key change shows as a key-change notice. +- **Revocation is expiry.** Certificates live a day, so a lost or replaced + key stops being vouched for within 24 hours without any revocation list. + For an immediate cut-off, the home relay marks the key revoked and + receiving relays and Stems that look up the sender see it; a sender + lookup happens on first contact and whenever a certificate names a key + different from the pinned one. +- **Relay key rotation.** A relay publishes the next signing key alongside + the current one for an overlap period (two DNS records); certificates + signed by either are accepted until the old one is removed. +- **Relay-to-relay limits.** A receiving relay rate-limits per issuing relay + as well as per sender, and a closed private relay accepts only its own + domains and named partner domains (see "A company can keep Stem mail + inside"). +- **Replay.** Envelope ids are unique and carry a timestamp; relays and Stems + drop duplicates and envelopes older than the 7-day transit cap. + +## The relay software + +Goal: easy to install, and dumb enough that Stem releases never touch it. + +### Build and install + +- **Go**, one static binary, plus a small Docker image. No Node, no npm. +- Built-in Let's Encrypt (autocert): point a domain at the box, run + `stem-relay init`, fill in the mail settings, start it. +- Storage is one embedded SQLite file through a pure-Go driver, so the binary + stays a single file and needs no database server. +- Lives in `relay/` in this repo as its own Go module, released on its own + tags (`relay-v1.0.0`). Stem's CI builds it and runs the mail tests against + the real binary. +- The protocol is a written spec plus shared test vectors (envelopes, + signatures, sender certificates) that both the Go relay and Stem's + TypeScript client test against. No shared code. + +### Dumb by design + +The relay sees only an envelope: + +``` +outer (relay reads): version · to · from · id · timestamp · size · + sender certificate · signature +inner (sealed): everything else: subject, body, attachments, + threading, mail kinds +``` + +Rules that keep Stem changes away from the relay: + +1. **The relay never looks inside the sealed part**, not even at its + version. Stems agree on the inner format between themselves; a Stem that + gets an inner version it doesn't know shows "this mail needs a newer + Stem" instead of failing. +2. **Key material is an opaque signed blob per address**, not named fields. + New key types or forward-secrecy prekeys are a new blob from Stems, not a + relay change. +3. **The protocol only grows.** Paths under `/v1/`, new fields optional, + unknown fields ignored on both sides. `/v1/info` lists what the relay + supports; Stem feature-detects and never needs the newest relay for core + mail. +4. **Policy is config, not code.** Every abuse number (invite budget, sender + and queue quotas, one knock, circuit-breaker thresholds, lookup limits) + lives in the config file. Tuning the global relay is a config edit. +5. **Optional features are switched in config.** Invites, complaint + webhooks and the circuit breaker are on for the global relay; a private + relay usually leaves them off. + +### What still forces an update + +- **Security fixes** in Go or a dependency. The image and binary check for + new versions and can update themselves; database migrations only go + forward. +- **Real protocol additions**, expected to be rare. The key log is the only + one planned, and it is a new endpoint. + +An ordinary Stem release changes nothing on the relay. + +## Abuse + +The relay can't read mail, so it can't filter content. Every defence works on +identity, counts and sizes. Numbers below are starting values, tuned on the +global relay; a private relay sets its own. + +### No mail without a key + +A sender can only seal to a key the relay returned, so there is nothing to +send to an unregistered or unlisted address, and the relay never stores mail +for one. An invite request (below) is the only thing a sender can ask for +such an address. + +### Verification emails + +The one email the relay sends, and the one a stranger can trigger by typing +someone else's address: + +- at most 3 codes per address per day, and per requesting Stem and IP; +- every verification email carries a "this wasn't me, never email me again" + link that puts the address on a suppression list (lifted only by the + owner verifying from that address later); +- registration needs a small proof-of-work from the Stem, so mass + registration costs CPU. + +### Invite emails + +Invites are the one way a sender makes the relay email a stranger, so they +get the tightest limits: + +- **An invite request counts as a knock.** It uses the sender's first-contact + allowance and the one-knock rule like a mail does. +- **Per recipient address: one invite email per 30 days**, whoever asks. + Further senders in that window are batched into the next one ("Vlado and + 2 others want to reach you on Stem"). After 3 invites with no + registration, the address gets no more. +- **Per sender: 5 invites per day, 20 per month.** Addresses verified in the + last 7 days can't trigger invites at all. +- **Fixed template.** The sender controls nothing in it but their verified + address; no free text, no display name, nothing to phish with. +- **Unsubscribe.** Every invite carries "never email me again", which puts + the address on the same suppression list as verification emails. +- **Bounces and complaints.** A hard bounce suppresses the address. A spam + complaint against an invite costs that sender their invite rights; repeat + complaints count as blocks (see below). +- **Global budget.** The relay sends at most a fixed number of invites per + day across all senders (start at 200, raise slowly as the sending + reputation builds). Invites over budget wait in a queue and expire with + the 30-day window; verification emails never wait behind invites. +- **Circuit breaker.** Invites pause automatically when, over a rolling 7 + days, hard bounces exceed 2% or spam complaints exceed 0.1% (Gmail's + bulk-sender line is 0.3%), and the operator is alerted. Verification + emails keep going. +- **Separate sending identity.** Invites go out from their own subdomain and + the provider's separate stream for marketing-like mail, never from + `awantech.sk` itself or the verification stream, with SPF, DKIM and DMARC + set up. If invites ever get blocklisted, company mail and verification + codes are unaffected. +- **Private relays send no invites** unless their operator turns them on; + colleagues are expected to be on the relay already. + +### One knock before acceptance + +A sender can have **one undelivered or unanswered mail** waiting for a +recipient who hasn't accepted them. Further mail to that recipient is +refused until the recipient accepts the Request, replies, or the knock +expires. The relay enforces this from metadata it already has (has this +recipient ever mailed this sender?). Strangers get one chance, not a flood. + +### Sender limits + +- New first-contact recipients per sender: 20 per day. +- New addresses (first 7 days after verification) get a quarter of the + normal limits. +- Total mails per sender per hour, regardless of recipient. + +### Queue quotas + +For mail held while a recipient is offline: + +- per mail: size cap (attachments included), 25 MB; +- per recipient: total held, 200 MB / 1,000 mails; past that a contact's + mail is refused as "recipient's queue is full", a stranger's is silently + dropped; +- per sender toward one recipient: a share of that queue, so one sender + can't fill someone's queue and block everyone else. + +### Blocks feed back to the relay + +Blocking a sender in Requests or a conversation tells the relay. The relay +then silently drops that sender's mail to that recipient for good. When many +distinct recipients block the same sender in a short window, the relay +throttles and then suspends that address, which can appeal by verifying +again. + +### Directory lookups + +Lookups need an authenticated, registered Stem and are rate-limited per Stem +(100 per day, burst 20), so bulk enumeration is slow and visible. They don't +hide who is listed; see Discoverability. + +### Known gap + +Free email providers make new identities cheap. Proof-of-work, new-address +limits, the one-knock rule and block feedback raise the cost; they don't stop +a determined spammer with many verified mailboxes. If that becomes real, the +next step is reputation (age, accepted contacts) gating first-contact sends. + +## Discoverability + +A key directory can't hide who is in it. Fake keys for unregistered +addresses were considered and rejected: a modified client can re-look up an +address and see the fake key become a real one when the person registers, +and with several accounts it can compare answers. Hiding membership +robustly would need something like private set intersection, which is out +of scope. So the design is honest instead, the way Signal is about phone +numbers: + +- **Listed (default).** Anyone who knows your address can learn that you use + Stem and send you one knock. That's what "people can reach me at my + address" means. +- **Unlisted.** Settings switch "Only people I've mailed can reach me". The + relay returns your key only to senders you've mailed first (it knows this + from delivery metadata). To everyone else a lookup answers exactly like an + unregistered address: same response, same timing, and no invite is sent. + Because "no key" covers both cases, an unlisted user is indistinguishable + from a non-user. +- **Registration replies identically** whether the address is new or taken + (see Identity and keys), so registering isn't a membership oracle either. +- **No status toward strangers.** A stranger sees only "sent", never + "delivered", "queue full" or "blocked". This hides blocks and the + recipient's activity, not membership. + +What leaks, accepted: listed users are discoverable one address at a time +(rate-limited); the relay operator knows who is registered; a +`_stem.` record shows a company uses Stem. + +## Encryption + +Each mail is sealed to the recipient's key and signed with the sender's key. +Attachments travel inside the sealed envelope, with a size cap. No forward +secrecy in v1: mail sits decrypted in both Stems' stores anyway, and a +ratchet can come later as a new envelope version. + +## Inbound + +- External mail lands in the Inbox addressed to `user`, badged as external, + with the sender's state: verified by email, verified by you, or key + changed. +- First mail from an unknown sender goes to **Requests**. The user accepts + (sender becomes a contact) or blocks (reported to the relay, see Abuse). +- **Incoming approval.** Every external mail is held from agents until the + user releases it, every time: first contact, accepted contacts, and new + replies inside a collaboration a persona is already working on. Until + then it is shown only to the user; no recall, capture, persona turn or + draft touches it. Releasing it is the user saying "my agents may act on + this". +- Releasing doesn't make it safe: a person can miss a prompt injection, so + the exit gating (Outbound) applies to every external-context turn + regardless. +- On a released mail the user can: + - reply by hand; + - ask a persona to draft a reply; + - bring it into an internal conversation with personas. +- External addresses are mail participants like `user` and persona ids. They + are told apart by the `@` (agent ids use `~`). + +## Outbound + +- A persona's `send_mail` to an external address creates a draft, never a + send. The draft shows in the conversation and in an Approvals view; the + user edits, sends or discards it. This reuses `MailApproval`. +- The user's own replies send directly. +- Approval is the main defence against prompt injection in external mail: + nothing leaves through mail without the user. + +### External conversations: full memory, gated exits + +A conversation that contains external mail is **external context**, and so +is every turn in it: the driver's, other personas', spawned agents', and any +internal conversation the user brings external mail into (the mark travels +with the mail). + +**What the agents get, by default:** everything a normal turn gets for +reading: recall, the user summary, standing answers, `read_past_chat`, +memory tools, and reading connected folders. That is what lets them answer +and coordinate well. Disclosure is controlled at the one place content +leaves, the outgoing mail, which the human approves. + +**What is gated, in code at `before_agent_start`, because approval doesn't +see it:** every other way out. External mail can carry a prompt injection, +and a turn with full memory is exactly the turn an attacker wants to make +leak through a side channel: + +- web reads and fetches always ask (a URL can carry data out); automatic + approval tiers are off for the turn; +- file and folder writes, `run_command`, coding agents, computer and browser + control, MCP tools, skill writes and `schedule_task` are removed; +- a persona can be granted specific ones back ("in external conversations + this persona may use: , "), default none. + +**What the approver sees:** the exact outgoing mail, body and every +attachment, not a summary of it. Later the approval may distinguish +**disclosure** ("may this be shared?") from **commitment** ("may Stem agree +to this or promise this work?"): "Sunday works for me" is not the same +approval as "I've booked Sunday". + +## Memory + +- Your agents read your full memory in external conversations (above). The + other Stem only ever receives approved mail; no memory, recall or files + are shared between Stems. +- Facts learned from external mail are tagged with that contact and recalled + only in conversations with that contact, so an outsider can't plant facts + that surface elsewhere. Capture of the owner's own messages in an external + conversation works as normal. +- Company memory, later, is a **team Stem**: a Stem on the company's private + relay with its own memory, skills and personas, which colleagues mail like + anyone else. Its owner can switch on auto-answer for members of that relay, + the one place where unattended replies make sense. Personal memory stays on + personal Stems. + +## Build notes (phase 1, 2026-10-14) + +Where the first build is stricter or narrower than the design above: + +- **Web is removed, not "ask".** A persona's draft runs unattended, so there + is nobody to ask; external-context turns get only the read-only allowlist + (`EXTERNAL_TOOLS` in `stem-mcp-extension.mjs`: read/grep/find/ls, memory + search, `read_past_chat`, `read_notes`, `send_mail`). New tools start closed. +- **No capture at all yet.** External-context turns are tainted out of Recall + entirely; contact-tagged facts are a later step. +- **Text only.** Attachments to and from other Stems aren't sent yet. +- **One counterpart per conversation.** An external conversation is with + exactly one address; personas join it only to draft. +- **Global relay keys** ship in the build for `relay.stem.awantech.sk` + (`GLOBAL_RELAY_KEYS` in `relay-client.ts`). A relay named by + `STEM_GLOBAL_RELAY_URL` brings its keys in `STEM_GLOBAL_RELAY_KEYS` or is + learned from its `/.well-known` on first use and pinned. + +## Phases + +1. **Transport.** Keypair, address registration with email codes, relay + (single binary, global and private), DNS routing, sealed delivery with + confirm-and-delete, external mail in the Inbox, Requests, hand-written + replies, sender certificates and relay signing keys, key pinning and + change notices, the unlisted switch, invite emails, the abuse limits + (verification and invite caps, one knock, sender and queue quotas, block + feedback, lookup limits). External-context exit gating lands here too, + since phase 1 already lets the user bring external mail to personas. +2. **Collaborations.** Persona drafts, the Approvals view (exact content and + attachments), the per-persona tool grants, contact-tagged facts, and + remote conversations shown as collaborations with a goal and pending + decisions. +3. **Teams and scopes.** Bounded collaborations ("coordinate this recording + with Martin; you may discuss availability, topics and past episodes") + where agents send routine replies within the scope without asking; + disclosure vs commitment approvals; private-relay member directory; the + team Stem with opt-in auto-answer for members; iOS support; the key log. + +## Open questions + +- Hosting of the global relay on the Awantech side. + +Resolved during the build: envelopes use Ed25519 signatures and an +X25519 + HKDF-SHA256 + AES-256-GCM seal (ChaCha20 is missing from Electron's BoringSSL) (no libraries beyond the Go +and Node standard crypto); the relay connection is SSE plus POST. Both are +specified in `relay/PROTOCOL.md`. +- Attachment size cap. +- Whether contacts and Requests sync to the phone in phase 1. diff --git a/docs/public-share-design.md b/docs/public-share-design.md new file mode 100644 index 00000000..1b5a3253 --- /dev/null +++ b/docs/public-share-design.md @@ -0,0 +1,163 @@ +# Public share links + +Status: built 2026-10-19 (relay, server, desktop, viewer, iOS); relay and +viewer live on share.heystem.app. + +## What it is + +From the Share button on an answer (chat message or mail reply, shown on +hover) the user picks what to share: the whole chat / mail thread, the +question and that answer, or the answer alone. Anyone with the link can open +it in a browser, without a Stem: + +``` +https://share.heystem.app/s/# +``` + +- **Snapshot, not live.** The link shows the chat as it was when shared. + Sharing again makes a new link; the old one keeps working until revoked. +- **Short-lived and capped (abuse).** A link works 30 days by default and + at most (7 and 1 day are offered); an address holds at most 100 working + links and makes at most 30 a day. The relay enforces all three. +- **Mail too, but only the user's own.** Mail with another person (inter-Stem) + and the memory check's conversation can't be shared; persona↔persona + consulting and approval cards are left out. +- **The relay never sees the content.** Stem encrypts the snapshot before + upload. The key rides in the URL fragment, which browsers never send to a + server. +- **Link previews, no indexing.** For the card chat apps show when the link + is pasted, the sharer publishes a title and a short description in the + clear: suggested from the snapshot (its title, the start of the first + answer), editable in the dialog, and switchable off. The relay fills the + viewer page's `og:` tags with it. Every page and response says `noindex, + nofollow, noarchive`; there is no robots.txt Disallow, since that would also + stop the crawlers that draw previews. +- **Only verified people publish.** Uploading needs a sender certificate for + an inter-Stem address (an email verified with a code). Every share is tied + to that address, so abuse can be traced and the address suspended. Viewers + stay anonymous; no login on the web. +- **Awantech hosts it** on the global relay box (EU DSA hosting provider): + every page has a report link; takedown by id; contact address on the page. + +## What is in a snapshot + +Built on the server from the same history the app shows (`readThread`), then +cleaned: + +| Kept | Dropped | +| --- | --- | +| User text, assistant MDX (charts, diagrams, tables, callouts) | System notices (failed/interrupted turns), failed sends | +| Generated images (embedded as data URLs) | Tool rows / activity (queries, paths, commands) | +| Web sources (public URLs) | Recall, file, mail and schedule context (already stripped from history) | +| The model's name (optional) | `` (personal memory), `` (dead in a static page) | +| User attachments: only when the user ticks "Include my pictures" | `stem://` and `file:` links (become plain text) | + +The user always sees a preview of exactly what will be published before the +upload. That preview is the main protection against leaking something private. + +Private chats can be shared: "private" means "not learned from", and sharing +is an explicit act. + +### Format (plaintext, before encryption) + +```json +{ "v": 1, "kind": "answer" | "thread", "title": "…", "sharedAt": 1760000000000, + "messages": [ + { "role": "user" | "assistant", "text": "…", "createdAt": 1760000000000, + "model": "GPT-6.1", "images": [{ "mime": "image/png", "dataUrl": "data:…", "alt": "…" }], + "sources": [{ "url": "https://…", "title": "…" }] } ] } +``` + +Viewers ignore unknown fields and show "made by a newer Stem" for `v > 1`. + +### Encryption + +``` +key = 32 random bytes +plain = gzip(JSON) +nonce = 12 random bytes +blob = 0x01 || nonce || AES-256-GCM(key, nonce, plain, aad = "stem-share-v1") +fragment = base64url(key), no padding +``` + +The browser decrypts with WebCrypto and `DecompressionStream('gzip')`. + +## Relay (global relay only, feature `share`) + +| Method & path | Auth | Body → response | +| --- | --- | --- | +| `POST /v1/shares` | SC | raw blob (`application/octet-stream`), `?ttl=` optional → `{id, url, expires}` | +| `GET /v1/shares/{id}` | — | → blob bytes; `404 not_found`, `410 gone` (revoked or taken down) | +| `DELETE /v1/shares/{id}` | SC, same address | → `{}` | +| `GET /v1/shares` | SC | → `{shares: [{id, size, created, expires}]}` for the certified address | +| `POST /v1/shares/{id}/report` | — | `{reason, detail?, contact?, key?}` → `202 {}` | + +- `id` = 16 random bytes, base64url (22 chars). +- Store: SQLite table `shares(id, owner, size, blob, created, expires, + removed, removed_why)`. Revoke/takedown keeps a tombstone (blob dropped) so + the link answers 410. Expired rows are swept. +- Limits in `[share]`: `max_bytes` (12 MiB), `max_ttl_days` (30, also the + default expiry), `max_live` (100 working links per address), + `per_address_day` (30), `quota_bytes` per address (500 MiB), + `reports_per_ip_hour` (20). +- Reports go to a `share_reports` table and the relay log. The optional `key` + is the reporter consenting to show us the content. +- CLI: `stem-relay share-reports`, `stem-relay share-takedown `. +- Closed-domain certificates are already refused by `certified()`. Stems + whose policy is anything but open never publish (the global relay is never + contacted under a restriction). + +## Viewer (`share.heystem.app`) + +- A separate origin from `heystem.app`, so shared content never runs where a + future web login would. +- Static build `vite.share.config.ts` → `dist/share`, reusing + `src/renderer/mdx` and a shared `ShareView` component (also used for the + in-app preview). Only web-safe imports: no `window.stem`. +- stem-relay serves the whole viewer origin from `[share] viewer_dir` + (only on the viewer host): `/s/*`, `/assets/*`, icons, `/v1/shares/*`. + Caddy proxies just those paths and 404s the rest of the relay API there. Strict CSP (`script-src 'self'`, + `connect-src 'self'`, `frame-ancestors 'none'`), `Referrer-Policy: + no-referrer`, `X-Robots-Tag: noindex`. +- Footer: "Made with Stem · heystem.app" and "Report". +- `/s/` is the built `index.html` with its `` block + replaced by the share's card. `og:image` is the + Stem icon (`/og.png`). + +## Stem server + +- `src/server/share/`: + - `snapshot.ts`: history → snapshot (the cleaning above). + - `seal.ts`: encrypt, build the link. + - `store.ts`: `shares.json` (id, link with key, thread, turn, kind, title, + dates) so the user can copy a link again or revoke it. +- `InterStemService.shareIdentity()`: the default verified address with a + fresh certificate; throws `no_address` / policy errors in plain words. +- IPC: `share:preview(source, opts)`, `share:create(source, opts)`, + `share:list()`, `share:revoke(id)`; `source = {kind:'chat', threadId} | + {kind:'mail', conversationId}`, `opts = {scope: 'thread'|'pair'|'answer', + anchorId, includeAttachments?, includeModel?, ttlDays?}`. + +## Desktop UI + +- The one entry point: a Share icon among an answer's hover actions (chat + message action row; a persona's reply in a mail thread). +- Dialog: what to share (whole chat / mail thread · question and answer · + answer only; question and answer preselected), rendered preview, toggles + (include my pictures, show model name), expiry (30 / 7 / 1 days), the link + preview card (editable title and description, drawn like an unfurl, with a + note that it is not encrypted), Create link. Then: the link with Copy, its end date, and Revoke. Without a verified address, the dialog explains and links to + Settings → Your email address. +- Settings → Your email address → "Shared links": list, copy, revoke. + +## Phase 2: iOS + +Share in the answer's context menu (chat and mail) with the same three +choices; calls `share:create` and opens the system share sheet. No preview renderer on the +phone at first, so the sheet shows what is included (counts, pictures off). + +## Not doing + +- Live links, comments, view counts. +- Server-side moderation of content (we can't read it). +- Per-recipient shares: that's inter-Stem mail. diff --git a/docs/stem-policy.md b/docs/stem-policy.md new file mode 100644 index 00000000..03127d12 --- /dev/null +++ b/docs/stem-policy.md @@ -0,0 +1,67 @@ +# Stem policy file (for administrators) + +A machine-wide file that limits what Stem may do on a company computer. The +user can't change it from inside Stem. Today it covers mail with other Stems. + +| System | Path | +| ------- | ------------------------------------------------- | +| macOS | `/Library/Application Support/Stem/policy.json` | +| Linux | `/etc/stem/policy.json` | +| Windows | `%ProgramData%\Stem\policy.json` | + +Make it writable by administrators only (root-owned, mode 0644). MDM tools +can deploy it as a plain file. A headless Stem server (Docker) takes the same +JSON in the `STEM_POLICY` environment variable; a file on the machine wins +over it. Stem reads the policy at start, so restart it after a change. + +## No mail with other Stems + +```json +{ "interStem": { "enabled": false } } +``` + +Settings shows "Turned off by your organization". Stem contacts no relay at +all, and phone push notifications don't go through the global relay either +(a server with its own APNs key still pushes directly). + +## Only inside the company + +```json +{ + "interStem": { + "domains": ["acme.com"], + "relay": "https://stem-relay.acme.com", + "relayKeys": ["k1:"] + } +} +``` + +- People can add only addresses in `domains`, and write only to addresses in + `domains`. Anything else fails at once with a message naming the policy. +- Every address in those domains is taken to live on `relay`, checked with + `relayKeys` (`stem-relay dns` prints them as the `k=` values). DNS is not + consulted, so a forged `_stem` record can't redirect anyone. +- The global relay is never contacted. Push notifications go through + `relay` (when it has `[push]` enabled) or not at all. +- Mail arriving from outside the domains is dropped without being shown. +- An address verified before the policy arrived (on the global relay, or + outside the domains) stops being used; adding it again moves it to the + company relay. + +To include a partner, list its domain too and leave out `relay` and +`relayKeys`: each domain's `_stem` DNS record then names its relay. A listed +domain without a record is unreachable; it never falls back to the global +relay. + +## When the file is wrong + +A file that exists but can't be read or parsed turns mail with other Stems +off (the reason is in Stem's log). A broken lockdown never fails open. + +## Pair it with the relay + +The policy controls company machines. The company relay controls who gets +in: set `closed = true` there and publish its `access=closed` DNS record +(`relay/README.md`, "Keep it inside the company"). With both, outsiders +can't reach your people and your people can't reach outsiders, even from a +Stem without the policy file. diff --git a/docs/user/browser-control.md b/docs/user/browser-control.md index f1c14a4f..76071282 100644 --- a/docs/user/browser-control.md +++ b/docs/user/browser-control.md @@ -5,39 +5,63 @@ the Stem extension: open pages, read them, click, type, fill in forms, upload fi download, and check a page's console and network requests. It works in background tabs, so the tab you are looking at never changes under you. -It runs on a Mac, in Arc, Chrome, Dia or Brave. It works whether Stem's server runs on -that Mac or somewhere else. +It runs on a Mac, in Arc, Chrome, Dia, Brave, Edge or Chromium; on Windows, in Chrome, Edge, Brave or +Chromium; and on Linux, in Chrome, Brave, Chromium, Edge, Vivaldi, Opera, or any other Chromium +browser, including AppImages such as Helium or Thorium. It works whether Stem's server runs on that computer or somewhere else. On Linux, +Snap and Flatpak builds of a browser can't reach Stem; use the browser's own .deb, .rpm or AppImage. +Run a new browser once before Set up, so Stem finds its settings folder. ## Setting it up -On the Mac whose browser Stem should use: +On the computer whose browser Stem should use: 1. Open Settings → Features → **Browser control** and switch on **Let Stem control this - Mac's browser**. + Mac's browser** (**this computer's browser** on Windows and Linux). 2. Press **Set up…**. Stem puts the extension in a folder, copies that folder's path, - brings your browser forward, and tells the browsers on this Mac how to reach Stem. + brings your browser forward, and tells the browsers on this computer how to reach Stem. The steps below then appear in Settings, with Copy and Open buttons. 3. In your browser, type the extensions page into the address bar (`chrome://extensions`, - or `arc://extensions` in Arc) and turn on **Developer mode**. + `arc://extensions` in Arc, or `edge://extensions` in Edge) and turn on **Developer mode**. 4. Click **Load unpacked**. In the folder picker press ⌘⇧G, paste the path with ⌘V, - press Return, then **Select**. + press Return, then **Select**. On Linux press Ctrl+L, paste with Ctrl+V, and press Enter. + On Windows click the folder picker's address bar, paste with Ctrl+V, press Enter, then + **Select Folder**. Settings shows the browser as connected within a couple of seconds. You do this once per browser; when Stem -updates, the extension updates itself. If the extension is installed in more than one -browser, a **Use** row picks the one Stem drives. When that browser is closed and a run -needs it, Stem starts it in the background. +updates, the extension updates itself. When that browser is closed and a run needs it, +Stem starts it in the background. On Windows and Linux it may open in front. On Linux, +Stem starts only Chrome, Brave, Chromium, Edge and AppImage browsers itself; open any +other browser, such as Vivaldi or Opera, before a run that needs it. + +## Several profiles + +If you keep separate browser profiles — an Arc profile for work and one for personal +things, say — load the extension in each one (press **Set up again** and repeat steps +3–4 from a window of the other profile). Each profile connects on its own, and Stem can +work in all of them: + +- **Profiles** in Settings lists them. Give each one a name, such as *Work* and + *Personal*. To tell which is which, click the Stem button in a browser's toolbar: it + says the name that profile goes by. +- **Use** picks the profile a task starts in. +- When you ask for something in a particular one ("in my work browser, …"), or the page + it needs is open there, the assistant switches to that profile and stays in it for + the rest of the task. Asked for the open tabs, it sees every open profile's tabs under + that profile's name. +- A profile you removed the extension from can be forgotten from the list once it is + closed. ## Who can use it The same two places as computer control, and a conversation gets it from exactly one: - **A persona** gets it from its browser pin: in the Personas tab, open the persona, - choose **Computer** on its **Runs as** tab, and pick a Mac under **Browser**. The + choose **Computer** on its **Runs as** tab, and pick a computer under **Browser**. The browser pin is separate from **Screen and apps**: a persona can have the browser without your screen. Pinned personas use it in chats, mail and scheduled tasks. - **A chat with no persona** gets it from Settings → Features → Browser control → - **Allow in chats**, either on a fixed Mac or letting the model pick one. + **Allow in chats**, either on a fixed computer or letting the model pick one. ## While it works @@ -60,17 +84,19 @@ still a page from the internet: keep an eye on runs that touch accounts that mat - **Uploads** take only files Stem already holds: attachments in the conversation, files in your Files place, files the assistant made, and files in connected folders. - It cannot upload an arbitrary file from the Mac. + It cannot upload an arbitrary file from your computer. - **Downloads** land in your browser's Downloads folder as usual, and a copy goes to the conversation's scratch folder on the server so the assistant can read or send it. ## When it does not work -- *"does not let Stem drive its browser"*: the switch on that Mac is off. +- *"does not let Stem drive its browser"*: the switch on that computer is off. - *"No browser with the Stem extension has connected"*: Set up has not been done there, or the extension was not loaded with Load unpacked. - *"the Stem extension did not connect"*: the browser started but the extension is disabled or removed; check its extensions page. -- *"not connected right now"*: the Mac is asleep or Stem is not running on it. +- *"not connected right now"*: the computer is asleep or Stem is not running on it. +- *"The browser profile … is not open"*: starting the browser opened a different + profile. Open a window of the one named, or ask for another profile. - Pages inside a cross-site frame (some payment and login widgets) are not part of what the assistant reads; a screenshot still shows them. diff --git a/docs/user/chats.md b/docs/user/chats.md index 6a209c78..937c47bb 100644 --- a/docs/user/chats.md +++ b/docs/user/chats.md @@ -29,14 +29,16 @@ there: while it runs. Once the answer is done the rows fold into one line, such as "Used 3 tools and 1 skill" with the total time. Click it to see every call again. - The mic button in the composer dictates. Speech is turned into text on the - device, and the audio never leaves it. It needs macOS 26 or iOS 26; elsewhere - the button is hidden. Pick the Mac's language in **Settings → App → Dictation**; - on the phone, hold the mic. + device, and the audio never leaves it. On a Mac it needs macOS 26, on the phone + iOS 26; on Windows and Linux it runs Whisper on your computer, downloaded the + first time you dictate. Pick the language in **Settings → App → Dictation** + (see [Dictation](settings.md#dictation)); on the phone, hold the mic. - **Web** turns web search on or off for the next message. It stays where you leave it, and is the same switch as **Settings → App → Web search**. Quick Chat keeps its own. - Web answers include cited sources: the sites' icons show under the answer; open them for a card per source. +- Places in an answer (an address, a shop, a venue) are links that open Maps. - The context meter shows how full the conversation is. - Ask for a picture and Stem draws it with your ChatGPT subscription (see [Settings → Image generation](settings.md#image-generation)). Click it to enlarge. @@ -50,16 +52,23 @@ there: - Use **+** on a folder to start a chat inside it. - Drag chats and folders to move or nest them. - ⌘-click (Ctrl-click) several chats, or Shift-click a range. A bar over the list - marks them read or unread, moves them to a folder, or deletes them after asking. + marks them read or unread, moves them to a folder, archives them, or deletes them + after asking. Dragging one of them moves them all. **Esc** clears the selection. -- Right-click a chat to mark it read or unread, **Write a subject**, **Rename**, - move, or delete it. Deletion is permanent. Tasks scheduled from the chat stay: - they run on their own and report by mail. +- Right-click a chat to mark it read or unread, **Archive** it, **Write a subject**, + **Rename**, move, or delete it. Deletion is permanent, so the confirmation offers + **Archive instead**. Tasks scheduled from the chat stay: they run on their own and + report by mail. +- **Archive** (`⌘⇧A`, `Ctrl+Shift+A`) moves a chat out of the list into the folded + **Archived** section at its foot. Nothing is lost: an archived chat still opens and + shows up in search, with an archive badge. Right-click it and choose **Move out of + archive** to bring it back, or just continue it: a new message brings it back on + its own. - Right-click a folder for **New subfolder**, **Settings…**, **Mark all as read** (every unread chat in it and its subfolders), **Rename** or **Delete**. Deleting a folder moves its chats and child folders to the parent. -- Use **Search**, `⌘F` on Mac, or `Ctrl+F` on Linux to search titles and message - text across every chat. +- Use **Search**, `⌘F` on Mac, or `Ctrl+F` on Windows and Linux to search titles + and message text across every chat. ### Folders that file chats for you @@ -75,6 +84,17 @@ Stem never moves a chat you placed yourself or a private chat. To undo a move, drag the chat back; it then stays wherever you put it. Filing runs on the same model as chat subjects (**Settings → Models → Quick tasks**). +### The menu bar + +The app's menu bar lists every action that has a shortcut, with its keys beside it, +so it is a good place to learn them. **File** has new chat, new mail, Quick Chat and +**Attach Files…**. **View** opens the side panel's tabs (**Chats**, **Memory**, +**Tools**, **Personas**, **Sources**, **Scheduled Tasks**). **Chat** has Stop, +effort, Fast, **Archive**, **Snooze…**, read or unread, and **Delete Chat…**. +**Window** lists the first nine chats under ⌘1–9 (Ctrl+1–9). **Settings** and +**Check for Updates…** are under **Stem** on a Mac and under **File** on Windows and +Linux. There is no Close Window item, so ⌘W (Ctrl+W) never closes Stem's window. + ## Private chats On a new, empty chat, click **Make this chat private** before sending. A private @@ -93,6 +113,8 @@ Hover over a message to reveal its actions: - **Delete from here**: remove that turn and everything after it. Requires a second click. - **Pin to this chat**: keep it on the chat's pinboard. +- **Share this answer as a link**: publish the chat, or part of it, as a link anyone + can open (see [Share links](share-links.md)). The pinboard is the strip under the chat title, **Pinned in this chat**. Besides whole messages, you can select a few words of a reply and click **Pin**, or type diff --git a/docs/user/connected-folders.md b/docs/user/connected-folders.md index ef07c0c1..199aa6eb 100644 --- a/docs/user/connected-folders.md +++ b/docs/user/connected-folders.md @@ -117,8 +117,9 @@ Connected folders tab groups it under this computer's name and shows when it las - After the first sync, changes sync within seconds while this computer is online; deletes and renames follow. The folder on this computer is always the original — nothing Stem does can change the mirror's contents from the server side. -- Mirrors skip `.git`, `node_modules`, OS junk files, symbolic links, and files over - 25 MB. Anything else skipped is counted on the folder's page. +- Mirrors skip `.git`, `node_modules`, OS junk files, whatever the folder's own + `.gitignore` files leave out, symbolic links, and files over 25 MB. Anything else + skipped is counted on the folder's page. - If the folder disappears (an unplugged drive, a rename), syncing freezes and the folder's row says so — the mirror keeps its last state rather than treating it as deleted. Disconnecting the folder in Stem is how you delete the mirror. diff --git a/docs/user/mail.md b/docs/user/mail.md index 02483916..7537fcc1 100644 --- a/docs/user/mail.md +++ b/docs/user/mail.md @@ -5,6 +5,9 @@ The first recipient coordinates the conversation and can consult the other personas. Replies arrive in your Inbox. A persona that makes pictures while answering (see [image generation](settings.md#image-generation)) attaches them to its reply. +This page is about mail with your own personas. To mail other people who use Stem, +see [Mail with other Stems](other-stems.md). + ## Built-in personas Stem ships with four personas. You can edit each one, but you cannot delete @@ -13,7 +16,7 @@ it. | Persona | What it does | Starts agents | Your memory | | --- | --- | --- | --- | | Normal | Answers on its own and brings in agents when a job needs them | yes | yes | -| Secretary | Delegates the work instead of doing it, schedules follow-ups, and mails you only decisions and results | yes | yes | +| Secretary | Delegates the work and keeps at it until the job is done, brings in any persona it needs without asking you, schedules follow-ups, and mails you only decisions and results | yes | yes | | Verifier | Checks the factual claims in the work it is sent and lists what is wrong | no | yes | | Critic | Reads what it is sent as its recipient would, without knowing who wrote it | no | no | @@ -35,7 +38,9 @@ persona**. Nothing changes until you press **Save**. Three tabs: - **Access** — **Private memory** (notes it keeps from its work; for a coding agent, **Standing answers** to the agent's recurring questions), **Sees your memory**, **Open to chats** (offered in the chat composer on the phone), **Can start - helpers** (see Agents below), **Uses every MCP server** (switch it off to pick which + helpers** (see Agents below), **Adds personas without asking** (it brings any + persona into a conversation it leads, including ones that work on your + computer; Secretary ships with it on), **Uses every MCP server** (switch it off to pick which servers it may use; the rest are hidden from it entirely), and **Send budget** (how many mails it may start between your sends). @@ -46,12 +51,18 @@ starts agents when a job needs checking or several hands. Mail **Secretary** for errands you want handled without the details, especially ones that need a follow-up later. It always hands the work to someone else, so -it adds a step that a direct question does not need. +it adds a step that a direct question does not need. It adds whichever personas +the job needs to the conversation itself, your coding, screen and browser +personas included, and does not ask you first. When a reply comes back +incomplete or stuck, it sends the work back or tries another persona instead of +handing the problem to you. It writes to you with the result, or when only you +can unblock it (a decision, a password, a payment). Normal is the one that stops +and suggests which persona to add. Mail **Verifier** or **Critic** directly when that is the whole job: check these claims, or read this draft cold. Mail a coding, screen or browser persona -directly when the request is that work. To let Normal or Secretary use one of -those, add it to the conversation yourself. +directly when the request is that work. Secretary brings those in on its own. +To let Normal use one of them, add it to the conversation yourself. ## Write and read mail @@ -69,16 +80,21 @@ and says who gets your reply. Each conversation keeps its own reply draft. While personas work, a live card shows who is on it and the latest steps, with **Stop**. Below it the newest mail is open and older ones fold to one line. Mail the personas sent each other folds onto the reply it led to ("Consulted Verifier · 2 -mails"). If you reply while they are still working, your mail is read when the -current run ends. A reply the server refused stays in the thread as **Not sent**, +mails"). If you reply while they are still working and only ask how it is going +("what's the status?"), the lead answers right away from their reports +and live work, and the work carries on undisturbed. A mail that also asks for +something is answered that way too and then read when the current run ends; any +other mail just waits for the run to end. A reply the server refused stays in the thread as **Not sent**, with Retry, Edit and Delete, so its text is never lost. **Forward** sends a mail on to other personas with a note of your own. The Inbox lists **Waiting on you** first, then **Working** (conversations you -sent that personas are still on), then the folded **Snoozed**, **Archived** and +sent that personas are still on), then **Requests** (first mail from people on +other Stems you haven't accepted yet), then the folded **Snoozed**, **Archived** and **Sent** sections. A run that needs your approval for a command waits in its mail with **Allow once** and **Deny**. You can also answer in words in the reply -box instead. On the Mac the reply and compose boxes have the dictation mic too. +box instead. Once you answer, the conversation moves out of **Waiting on you** to +**Working**. The reply and compose boxes have the dictation mic too. ## Agents @@ -110,7 +126,8 @@ agent can fill only five of those places. The last one is kept for the persona you mailed, so it can always have its lead's work checked. An agent never gets integrations its starter lacks, and a persona that works on your computer (a coding agent, screen or browser control) is started as an agent -only in a conversation you added it to. +only in a conversation you added it to, or one led by a persona with **Adds +personas without asking** on. Orchestrator is no longer a built-in persona, because every persona that can start agents now does its job. If you had edited Orchestrator's prompt, it @@ -122,8 +139,10 @@ A scheduled task that finds something worth telling you sends mail. All firings of one task share one conversation. Each firing keeps its own headline, and the conversation takes the latest headline as its subject. Beneath the short notice the mail carries the run's full reply, such as a report or the drafts it wrote, -once the run finishes. Every run happens in a thread of its own that no chat -shows; the mail, and the Work beneath it, is the only place a run appears. A +once the run finishes. A task scheduled from a mail conversation, such as a +follow-up check a persona set up, reports in that same conversation instead, so a +chain of checks stays one Inbox row. Every run happens in a thread of its own that +no chat shows; the mail, and the Work beneath it, is the only place a run appears. A task that starts failing sends one mail with the reason. Pictures a run makes come with its mail, and a run that made pictures always sends one. diff --git a/docs/user/other-stems.md b/docs/user/other-stems.md new file mode 100644 index 00000000..b1f29acb --- /dev/null +++ b/docs/user/other-stems.md @@ -0,0 +1,127 @@ +# Mail with other Stems + +Maya's client Tomas uses Stem too. He mails her from his Stem to ask when the +next release can ship. His mail waits for Maya alone. She lets her agents read +it, asks Normal to draft a reply from what it knows about her projects, edits +the draft and sends it. Tomas's Stem never sees her memory, only the mail she +sent. + +Mail between Stems goes to a person's real email address, on their own Stem. +It is encrypted from one Stem to the other. Your personas can prepare replies, +but every mail that leaves is one you sent. + +## Add your email address + +1. Open Settings → Features → **Your email address**. +2. Type your address under **Add an address** and press **Email me a code**. +3. Type the code from that email and press **Confirm the code**. If you clicked + the link in the email instead, press **I used the link in the email + instead**. + +The address then shows **Connected**. You can add more than one; mail you start +goes out from the first. The trash button (**Stop using this address with other +Stems**) removes one. + +Once an address is verified, two more rows appear: + +- **Only people I've written to can reach me.** With it on, everyone else + can't tell you use Stem at all: to them your address looks like one that + isn't on Stem. +- **Your key.** A short fingerprint. Compare it with someone in person to + verify each other (see Keys below). + +## Write to someone + +In **New mail**, press **To a person** and type their email address in **To**. +They need Stem too, with that address verified. **Send** sends it at once: it +is your own mail, so there is nothing to approve. Your replies in that +conversation go straight to them as well. + +Each mail shows how it went: **Sending…**, **Sent**, **Delivered**, **Not +collected in 7 days**, **Not sent**, **Can't be reached**, or **Waiting — their +key changed**. **Can't be reached** means they aren't on Stem, or they only +accept mail from people they've written to; the button next to the message asks +Stem to email them an invite, which carries none of your mail. A mail the +relays can't take is retried for up to a week. + +A conversation with another person has one counterpart. You can't forward its +mail to personas, add personas to it, or share it as a public link. + +## Mail you receive + +- **Requests.** The first mail from someone new waits under **Requests** in the + mail list, out of the Inbox. Open it and press **Accept** to make them a + contact, or **Block**. Someone you wrote to first is a contact already. +- **Held mail.** Every mail from another person arrives held, including mail + from contacts: *Only you can see this. Your agents can't read it until you + let them.* No persona, memory capture or recall touches it. **Let my agents + read it** releases that mail and the earlier ones in the conversation. + +## Drafts by your personas + +Under the reply box, pick a persona, optionally say what the reply should say, +and press **Draft a reply**. It works only after you have let your agents read +their mail. The persona drafts from your memory and connected folders; that is +what makes the draft useful. + +The draft appears as **Send this to …?** with the exact text they will receive. +Edit it if you like, then press **Send** or **Discard**. Writing your own reply +instead sets the draft aside. + +While a persona drafts for someone outside, it can only read: your memory, past +chats, notes and files. Web access, commands, writing files, MCP servers, coding +agents, computer and browser control, skills and scheduling are all switched off +for that turn, so a mail written to trick your assistant has no way to send +anything out. A drafting turn is also kept out of your memory. + +## What the other side sees + +Only the mails you send. Their Stem and their personas never get your memory, +your chats or your files. Mail they receive from you is held for them the same +way yours is for you. + +## Keys + +Each Stem has its own key. Your contacts are listed under **Contacts** in +Settings → Features → **Your email address**, each marked **Verified by email** +or **Verified by you**, with their key. To check a contact, compare keys with +them in person, then press the shield button (**I compared this key with them +in person**). + +If a contact's key changes, the conversation says *their key changed — a new +device, or someone else* and nothing goes to them until you decide. Check with +them another way, then press **It's them — use the new key**. Mail that waited +on the change then goes out to the new key. A contact with a new key counts as +checked by email only, until you compare again. + +**Block** (in Requests, or the block button in **Contacts**) drops their mail +without telling them. Blocked people are listed under **Blocked**, where you +can unblock them. + +## Relays and encryption + +Mail travels through a relay. The global one is `relay.heystem.app`; a company +can run its own for its domain. Each mail is sealed to the recipient's key +before it leaves your Stem, so the relay can't read the subject or the body. It +does see who mails whom, when, and how big the mail is. It holds a mail only +until the other Stem collects it, for at most 7 days. + +## Company restrictions + +An administrator can limit this with a policy file on the computer (see +[the policy file](../stem-policy.md)). You can't change it from Stem. + +- **Turned off by your organization**: no mail with other Stems and no public + links. Stem contacts no relay at all. +- **Limited by your organization**: you can add and write to addresses only in + the listed domains, through the company's relay. Mail from outside those + domains is dropped without being shown, and public links are off. + +A company relay can also be closed, so outsiders can't reach its people even +from a Stem without the policy file. + +## Not yet + +- Attachments: mail to and from other Stems is text only. +- The iPhone: Requests, letting your agents read a mail, and persona drafts are + on the desktop app. diff --git a/docs/user/quick-chat.md b/docs/user/quick-chat.md index 3f9f883a..1ec823e0 100644 --- a/docs/user/quick-chat.md +++ b/docs/user/quick-chat.md @@ -37,5 +37,43 @@ again or click the pill. - **Note**—or starting with `/note ` or `//`—is the optional direct-save path for Memory. No chat turn. Memory must be on; credential-like text is not saved. +On Linux under GNOME on Wayland, the pill sits in the bottom-left corner only with +Stem's GNOME extension (**Settings → Features → Computer control**); without it, GNOME +places the pill itself. Linux also gets a tray icon with **Summon Quick Chat**, +**Open Stem** and **Quit Stem**. Stock GNOME shows tray icons only with the +AppIndicator extension. + +## Ask about this window + +Maya is in Slack and Jana has asked her to decide something. She presses her +**Ask about this window** shortcut. Quick Chat opens beside the Slack window, +not on top of it, with a chip showing which window it captured. She asks +“What does Jana need from me?” and the answer appears right there. + +- The chip shows a picture of the window. Its text goes along too, so the + assistant reads names and messages exactly. Click **×** on the chip to ask + without the window. +- If an MCP server covers the app, a second chip names it. The assistant uses it + to read what the window doesn't show and to act in the app, and shows you + what it will send before sending. See + [Works with apps](tools/mcp-servers.md#works-with-apps). +- With computer control allowed for this computer (**Settings → Features**), the + assistant can also click and type in that window. +- Nothing is captured until you press the shortcut. +- The panel sits right of the window, else left, else below or above. It covers + the window only when the window fills the screen. It follows the window if you + move it. +- Pressing it again in the same window continues that conversation. In another + window it starts a new one. +- On a Mac it needs **Screen Recording** for the picture and **Accessibility** + for the text. If one is off, the chip says which and opens the right System + Settings pane. On Windows it needs no setup. On Linux it needs the GNOME + extension from **Settings → Features → Computer control**. + +Set the shortcut under **Settings → Quick Chat → Ask about this window**. It is +off until you record one. On GNOME (Wayland) apps can't set global shortcuts: +copy the command shown there and add it as a custom shortcut in GNOME's +Keyboard settings. + Quick Chat has its own thread. Set its defaults in [**Settings → Quick Chat**](settings.md). diff --git a/docs/user/scheduled-tasks.md b/docs/user/scheduled-tasks.md index 0a855b9d..0fc6024b 100644 --- a/docs/user/scheduled-tasks.md +++ b/docs/user/scheduled-tasks.md @@ -26,6 +26,10 @@ arrives as mail in the Inbox. - Stem must be running. If Stem is closed or the computer sleeps, each overdue task runs once when Stem can run again. +- If the server stops in the middle of a run, the run is recorded as cut off by a + server restart and runs once more when Stem starts again. A run cut off twice in a + row is not retried; it waits for its next scheduled time, and the task's row says + why. - Times use the local time of the machine Stem runs on: your computer, or your server if you moved Stem to one. - Each run starts with an empty context: the task’s prompt, the persona it runs @@ -42,7 +46,8 @@ arrives as mail in the Inbox. - Runs can use enabled tools. Web-search tasks request search automatically. Native search needs a compatible model; otherwise enable a search-capable tool. - A run that has something for you sends **mail**: all firings of one task share - one conversation in the Inbox, and the run’s full reply is attached beneath the + one conversation in the Inbox (a task scheduled from a mail conversation reports + in that conversation), and the run’s full reply is attached beneath the notice once it finishes. Each run is also shown what the task's earlier runs already mailed, so a watch task reports a finding once rather than every morning. A run that finds nothing leaves no trace, and no run ever makes a chat unread. diff --git a/docs/user/settings.md b/docs/user/settings.md index 063143ac..c7ed1239 100644 --- a/docs/user/settings.md +++ b/docs/user/settings.md @@ -14,10 +14,10 @@ instructions.

Settings has four tabs: **App** (appearance, the conversation, Quick Chat, dictation, -the Escape key, notifications, About), **Features** (mail among personas, image -generation, commands, coding agents, computer control, browser control), **Server** -(where Stem runs, your devices, moving or backing up) and **Models** (providers, which -model does which job, memory recall, web search). +the Escape key, notifications, About), **Features** (mail among personas, your email +address, shared links, image generation, commands, coding agents, computer control, +browser control), **Server** (where Stem runs, your devices, moving or backing up) and +**Models** (providers, which model does which job, memory recall, web search). ## Providers and models @@ -226,7 +226,8 @@ also left behind when you move Stem to another machine. ## Coding agents and computer control A coding agent (Claude Code, Codex, OpenCode, …) takes programming work off Stem's -hands; computer control lets Stem see a Mac's screen and click and type there. Both +hands; computer control lets Stem see a computer's screen (Mac, Windows, or Linux with +GNOME) and click and type there. Both come from two places, and a conversation gets them from exactly one: - **A persona** gets them from its own setup, on the **Runs as** tab of its editor in @@ -257,7 +258,8 @@ another computer stops to ask you which folder. A computer that is asleep or has Stem closed fails right away with a message saying so; nothing is queued. Each computer still has to agree for itself: **Run coding agents on this computer** -and **Let Stem control this Mac**, in the same section *on that machine*. Command +and **Let Stem control this Mac** (or **this computer** on Windows and Linux), in the +same section *on that machine*. Command approvals apply to what a coding agent runs, the same as for personas. A persona that drives Claude Code can instead switch on **Approves its own actions**: Claude Code's own check then decides each step, so routine work never waits on a card, and Stem shows no @@ -266,10 +268,36 @@ cards and keeps no approval log for that persona. If Stem says it cannot use a coding agent or control the computer, it names the reason: the chat runs as a persona without that setup, or chats are switched off here. +On Linux, computer control needs GNOME on Wayland. Once you switch it on, a +**GNOME extension** row appears: **Install** puts Stem's GNOME Shell extension in your +extensions folder. GNOME loads a new extension only at login, so log out and back in +once; the row then says **Running**. Later updates to the extension arrive without +logging out. You can turn the extension off in GNOME's Extensions app. Two Wayland limits apply when Stem works inside one window instead of +the whole screen: that window gets no key presses, only text typed into a field and +buttons pressed by name, and some GTK 4 lists can be selected but not opened. Working on +the whole screen has no such limits. + +On Windows, computer control works on Windows 10 and 11 with nothing to install. +Working inside one window, Stem can type and press plain keys there, but not Ctrl, +Alt or Shift combinations; it uses the window's menus instead, or the whole screen. + **Browser control** follows the same rules with its own pin and its own switch: a persona's **Browser** (Runs as → Computer), or Settings → Features → Browser control → -Allow in chats, plus **Let Stem control this Mac's browser** on the Mac itself. Setting -up the extension is described on the [Browser control](browser-control.md) page. +Allow in chats, plus **Let Stem control this Mac's browser** (**this computer's +browser** on Windows and Linux) on that computer itself. Setting up the extension is +described on the [Browser control](browser-control.md) page. + +## Your email address + +Settings → Features → **Your email address** is the address other people's Stems +reach you at. Type it in and Stem emails you a code to confirm it is yours. Once it +is confirmed, you can [mail people who use Stem](other-stems.md), and your [share +links](share-links.md) carry it. **Only people I've written to can reach me**, +**Your key** and your **Contacts** appear here too. + +If your organization runs Stem for you, this section may say **Turned off by your +organization** (no mail with other Stems and no public links) or **Limited by your +organization** (only addresses at its domains, through its relay). ## Image generation @@ -306,10 +334,18 @@ Under **Settings → App → Appearance**, set per computer: ## Dictation -The mic button in the composer turns speech into text with the device's own model; -the audio never leaves it. **Settings → App → Dictation → Language** picks the Mac's -language, or **Automatic**. A language's model downloads the first time you use it. -It needs macOS 26; on the phone (iOS 26), hold the mic to pick the language. +The mic button in the composer turns speech into text on your own computer; the +audio never leaves it. **Settings → App → Dictation → Language** picks the language, +or **Automatic**. + +- **On a Mac** it uses the Mac's own speech model and needs macOS 26. A language's + model downloads the first time you use it. +- **On Windows and Linux** it runs Whisper on the computer's processor (Beta). The + model downloads the first time you dictate. **Speech model** picks **Fast** (a + 240 MB download, quick on any computer, best in English) or **Accurate** (a 730 MB + download, slower, much better in other languages). **Language** lists 38 + languages, Slovak and Czech among them. +- **On the phone** (iOS 26), hold the mic to pick the language. ## Escape key @@ -365,8 +401,9 @@ server](../running-on-a-server.md) for setting one up. **Devices** lists everything signed in to this Stem, with when each was last seen. **Withdraw** signs one out; Stem warns first if MCP servers run on it. **Add a device** names the new one and gives a code to enter there (**Get a code**), or **Pair a phone** -shows a QR code to scan in the Stem iPhone app (beta). The phone app has chats, mail, -rich answers, dictation and approvals, against the same Stem. +shows a QR code to scan in the Stem iPhone app (beta). The phone app has chats in their +folders, mail, rich answers, dictation, approvals and share links, against the same +Stem. **Move or back up this Stem** is described in [Moving and backups](moving-and-backups.md). @@ -374,10 +411,12 @@ rich answers, dictation and approvals, against the same Stem. Shows the version you're running — worth quoting when you report a problem. -**Updates** says whether a newer Stem exists and what to do about it. On Linux -(the AppImage) a new release downloads in the background and installs itself the -next time you start Stem — or right away with **Restart now**. On a Mac, **Get -the update** opens the release page and you install it the way you did the first +**Updates** says whether a newer Stem exists and what to do about it. The Linux +AppImage, the Windows installer, and a Mac with Stem in its Applications folder +download a new release in the background and install it the next time you start +Stem, or right away with **Restart now**. The Linux .deb, and a Mac running Stem +from anywhere else (straight off the disk image, say), only tell you: **Get the +update** opens the release page and you install it the way you did the first time. **Check now** asks immediately; **Check for updates automatically** turns the periodic check off entirely. The check asks one question of GitHub, where Stem's releases live, and sends nothing about you or your chats. diff --git a/docs/user/share-links.md b/docs/user/share-links.md new file mode 100644 index 00000000..690bdecf --- /dev/null +++ b/docs/user/share-links.md @@ -0,0 +1,102 @@ +# Share links + +Maya asked Stem to compare three ways to deploy her app, and a friend wants to +read the answer. She shares the question and answer as a link and pastes it in +a chat. Her friend opens it in a browser, without Stem. + +A share link is a copy of the conversation as it was when you shared it. Later +messages are not added. Sharing again makes a new link. + +## Before you start + +Sharing needs a verified email address, so every public link traces back to a +real person. Add one under Settings → Features → **Your email address** (see +[Mail with other Stems](other-stems.md)). Without one, the Share dialog says so +and offers **Verify an address in Settings**. + +## Share an answer + +- **In a chat**, hover an answer and click the share button (**Share this + answer as a link**). +- **In mail**, click the share button on a persona's reply (**Share as a + link**). +- **On the iPhone**, touch and hold an answer or a persona's reply and choose + **Share as Link…**. + +The dialog **Share as a link** opens. Pick what to share: + +- **Whole chat** (**Whole mail thread** in mail): every message. +- **Question and answer**: the answer and the question before it. This is the + default. +- **Answer only**. + +Below it is a preview of exactly what will be published. Check it: anyone with +the link can read what you see there. Then: + +- **Show which model answered** is on by default. The model shows by name, such + as `gpt-6.1-sol`. +- **Include my pictures** appears when you attached pictures. They are left out + unless you tick it. +- Expiry: **Expires in 30 days** (the default), 7 days or 1 day. + +Press **Create link**. Stem copies the link to the clipboard and shows it with +**Copy link**, **Open in the browser** and **Revoke**. + +## What is left out + +Tool steps, memory cards and in-app links are left out. Links to places inside +Stem keep their words as plain text. Suggested replies, notices about failed +turns and messages that failed to send stay out too. In mail, only the mails +between you and your personas are shared: the personas' mail to each other and +approval cards stay out. Pictures Stem made are included, and so are web +sources. + +A private chat can be shared. Private only means Stem doesn't learn from it. + +## The preview card + +Chat apps show a card when a link is pasted. **Show a preview when the link is +pasted in a chat app** is on by default, with a title and description Stem +suggests from the conversation. Edit them before you create the link, or switch +the card off. Unlike the page, this preview is not encrypted: the share server +and anyone the link reaches can read it. Search engines are told not to index +any of it, page or card. + +## Encryption + +Your Stem encrypts the page before it uploads it to `share.heystem.app`. The +key is the part after `#` in the link. Browsers never send that part to a +server, so the share server can't read the page, and only people you give the +whole link to can. A link missing that part opens as *This link is incomplete*. +The link is published under your verified address. + +## Limits + +- A link works for at most 30 days. +- You can have at most 100 working links at a time, and make at most 30 a day. +- A share that is too big fails with a note to leave out the pictures or share + one answer instead of the whole chat. + +## Take a link down + +Settings → Features → **Shared links** → **Public links you made** lists your +working links with what they share and their end date. The section appears once +you have made a link. Each row has **Copy link** and a revoke button (**Revoke — +the link stops opening**). A revoked link opens as *This link was turned off*. +Anyone who already saved the page keeps their copy. + +Each shared page also has a **Report** button for readers. A link taken down after a +report opens as *This link was removed*. + +## What can't be shared + +- Mail with another person (see [Mail with other Stems](other-stems.md)). Their + mail is theirs to publish. +- The Memory tab's deep check conversation. + +## Company computers + +When a company [policy file](../stem-policy.md) turns off or limits mail with +other Stems, sharing is off too: *Your organization has turned off sharing +outside the company.* It is also off when your address is on a company relay +that keeps its people inside. diff --git a/docs/user/shortcuts.md b/docs/user/shortcuts.md index cf6d4322..d4e78b84 100644 --- a/docs/user/shortcuts.md +++ b/docs/user/shortcuts.md @@ -24,6 +24,7 @@ Everything Stem binds to a key, and a few composer tricks that no keycap adverti | **Mark read or unread** — flips the unread mark on the selection | ⌘⇧D | Ctrl+Shift+D | | **Stop** — interrupts the answer in progress | ⌘. | Ctrl+. | | **Delete chat** — asks to delete the open chat; press again to confirm | ⌃X | Ctrl+Shift+X | +| **Settings** — opens Settings in the right-hand panel | ⌘, | Ctrl+, | | **Send** — sends the message; Shift+Enter starts a new line instead | ⏎ | Enter | ## Tricks diff --git a/docs/user/tools/mcp-servers.md b/docs/user/tools/mcp-servers.md index 71a80a5e..5ad6c620 100644 --- a/docs/user/tools/mcp-servers.md +++ b/docs/user/tools/mcp-servers.md @@ -66,6 +66,18 @@ or headers, and the OAuth client), **Runs on** (when you have computers to choos from), and **Status**. Stored secrets show as dots; leave them to keep them, or type a new value. **Save** writes everything at once; **Cancel** throws the edit away. +## Works with apps + +On a Mac, [**Ask about this window**](../quick-chat.md#ask-about-this-window-mac) +uses the server that covers the window's app. The model is told to prefer it +for reading and acting in that app, and to show you what it will send before +sending. + +By default a server covers apps named like it: `slack` covers Slack, +`linear-mcp` covers Linear. To choose the apps yourself, open the server and +fill in **Works with apps** with app names or bundle ids, separated by +commas. Clear the field to go back to matching by name. + ## Where a server runs **Command** and **URL** say how Stem talks to a server, not where it runs. A server diff --git a/docs/windows-dev.md b/docs/windows-dev.md index e7183960..73f1bb5a 100644 --- a/docs/windows-dev.md +++ b/docs/windows-dev.md @@ -1,9 +1,10 @@ # Windows development (experimental) -Stem’s released installers target macOS and Linux. **Windows is a terminal-first -dev port**: you can clone, install, and run from a user account **without admin -rights**, using a portable Node.js zip. Packaging (NSIS/portable exe) is not -included yet. +Releases from 0.7.0 on include a Windows installer, `Stem-Setup-.exe` +(per user, no admin prompt, updates itself; unsigned, so SmartScreen asks once). +This page is about running Stem from a clone: you can clone, install, and run +from a user account **without admin rights**, using a portable Node.js zip. +`npm run dist:win` builds the installer on a Windows machine, into `release\`. Personal data still lives outside the clone, under `%APPDATA%\Stem\` (and `%APPDATA%\Stem Profiles\` for `--fresh` / `--profile=`). Reinstalling Node or diff --git a/electron-builder.yml b/electron-builder.yml index 16c617e2..a1b1c33c 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -100,9 +100,10 @@ files: # whole repository (src/, tests/, mobile/, docs/, Dockerfile, …) inside the app # because of two such lists. Target-specific trimming lives in the hook instead. afterPack: scripts/prune-onnxruntime.mjs -# Compiles native/mac/stem-computer (the computer-control persona's Swift -# helper) before the mac app is packed; a no-op on every other platform. -beforePack: scripts/build-mac-helper.mjs +# Compiles the computer-control helper before the app is packed: the Swift one +# (native/mac/stem-computer) for a Mac, the C# one (native/windows/stem-computer) +# for Windows; nothing for Linux. +beforePack: scripts/build-native-helpers.mjs mac: target: [dmg, zip] @@ -132,6 +133,11 @@ mac: linux: target: [AppImage, deb] executableName: stem + # Computer control on GNOME: the gjs helper and the Shell extension it talks + # to (installed from here into ~/.local/share/gnome-shell/extensions). + extraResources: + - from: native/linux + to: linux # 1024px source; electron-builder derives the icon size set. icon: build/icon.png category: Utility @@ -140,6 +146,26 @@ linux: entry: StartupWMClass: Stem +win: + target: [nsis] + # Derived from the 1024px png (electron-builder writes the .ico). + icon: build/icon.png + extraResources: + - from: build/native/stem-computer.exe + to: bin/stem-computer.exe + # No code-signing certificate yet: SmartScreen warns on the first download + # ("More info → Run anyway"). electron-updater checks a signature only when + # publisherName is set, so updates still apply. + +# Per user, into %LOCALAPPDATA%\Programs\Stem: no UAC prompt to install, and +# the updater can run the next installer over it without one either. +nsis: + oneClick: true + perMachine: false + artifactName: Stem-Setup-${version}.${ext} + shortcutName: Stem + deleteAppDataOnUninstall: false + deb: # Expose `stem` on PATH so DE keyboard shortcuts can run `stem --quick-chat` # (the Wayland summon path — see the single-instance handler in server/index.ts). diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 03e15aef..8c5540fe 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -67,7 +67,10 @@ export default defineConfig({ // The browser extension's native-messaging host: plain Node, run by // the Stem binary with ELECTRON_RUN_AS_NODE from the wrapper script // Set up writes (desktop/browser-host/install.ts). - 'browser-native-host': 'src/desktop/browser-host/native-host-main.ts' + 'browser-native-host': 'src/desktop/browser-host/native-host-main.ts', + // Dictation off the Mac: Whisper in its own utilityProcess + // (utilityProcess.fork(dist/main/dictation-worker.js) in desktop/dictation/whisper.ts). + 'dictation-worker': 'src/desktop/dictation/whisper-worker.ts' }, // transformers.js must stay external: it lazily loads onnxruntime-node's // native .node binary, which cannot live inside a rollup bundle. Resolved diff --git a/eslint.config.js b/eslint.config.js index e1102cae..037c0290 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -67,6 +67,17 @@ export default [ languageOptions: { ecmaVersion: 'latest', sourceType: 'module' }, rules: { 'no-undef': 'off' } }, + { + // The Linux computer-control helper and GNOME Shell extension run under + // gjs, whose globals (global, print, printerr, TextEncoder…) eslint does + // not know. + files: ['native/linux/**/*.js'], + languageOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + globals: { global: 'readonly', print: 'readonly', printerr: 'readonly', TextEncoder: 'readonly', TextDecoder: 'readonly', console: 'readonly' } + } + }, { // Tests: Playwright's `use()` fixture API trips react-hooks/rules-of-hooks // (it's not a React Hook), and probing the renderer through `window.stem` diff --git a/experiments/browser-control/results/2026-10-05-extension.json b/experiments/browser-control/results/2026-10-05-extension.json index 2b8c8df3..f3791bd0 100644 --- a/experiments/browser-control/results/2026-10-05-extension.json +++ b/experiments/browser-control/results/2026-10-05-extension.json @@ -1,7 +1,7 @@ { "headless=new": { "recordedOn": "2026-10-05", - "recordedAt": "2026-10-05T16:19:30.832Z", + "recordedAt": "2026-10-19T19:05:23.514Z", "display": { "asleep": false, "locked": false @@ -11,152 +11,152 @@ "executable": "/Users/join3r/Library/Caches/ms-playwright/chromium-1228/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", "extensionId": "bbadcpfklblgjocjcaemcmojkikjnoao", "status": "passed", - "passed": 28, + "passed": 29, "failed": 0, "checks": [ { "name": "hello: protocol 1, the manifest-derived id, version", "ok": true, "ms": 0, - "detail": "bbadcpfklblgjocjcaemcmojkikjnoao v0.6.0" + "detail": "bbadcpfklblgjocjcaemcmojkikjnoao v0.6.0, profile 265d99de…" }, { "name": "tabs lists the user’s own tab as active", "ok": true, "ms": 2, - "detail": "1 open tab. \"active\" is the tab showing in its window — the user may be looking at it; the rest are in the background. Pass an id as `tab` to work in it.\nWindow 1 (last focused: new tabs open here):\n- 442773629 · The user’s own page — http://127.0.0.1:58345/user (active)" + "detail": "1 open tab. \"active\" is the tab showing in its window — the user may be looking at it; the rest are in the background. Pass an id as `tab` to work in it.\nWindow 1 (last focused: new tabs open here):\n- 1293553521 · The user’s own page — http://127.0.0.1:49512/user (active)" }, { "name": "open: new background tab, the user’s tab stays active", "ok": true, - "ms": 150, - "detail": "Opened tab 442773630 in the background: Fixture form — http://127.0.0.1:58345/form?tab=F\n2 open tabs. \"active\" is the tab showing in its window — the user may be looking at it; the rest are in the background. Pass an id as `tab` to work in it.\nWindow 1 (last focused: new tabs open here):\n- 442773629 · The user’s own page — http://127.0.0.1:58345/user (active)\n- 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F (yours, your current tab)" + "ms": 149, + "detail": "Opened tab 1293553522 in the background: Fixture form — http://127.0.0.1:49512/form?tab=F\n2 open tabs. \"active\" is the tab showing in its window — the user may be looking at it; the rest are in the background. Pass an id as `tab` to work in it.\nWindow 1 (last focused: new tabs open here):\n- 1293553521 · The user’s own page — http://127.0.0.1:49512/user (active)\n- 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F (yours, your current tab)" }, { "name": "snapshot: outline with refs, same-origin iframe spliced in", "ok": true, - "ms": 13, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\n- banner\n - navigation \"Main\"\n - link \"Page two\" [ref=e1] → /page2\n - link \"Open help\" [ref=e2] → /help\n - link \"Download report\" [ref=e3] → /download\n- main\n - heading \"Order form\" [level=1] [ref=e4]\n - paragraph: Fill in the form below and press Greet.\n - textbox \"Name\" [ref=e5]: Ada\n - textbox \"Email\" [ref=e6]\n - textbox \"Password\" [ref=e7]\n - combobox \"Colour\" [ref=e8]: Green\n - option \"Red\"\n - option \"Green\" [selected]\n - option \"Blue\"\n - checkbox \"Subscribe\" [ref=e9]\n - group \"Size\"\n - radio \"Small\" [checked] [ref=e10]\n - radio \"Large\" [ref=e11]\n - textbox \"Notes\" [ref=e12]: hello\n - button \"Greet\" [ref=e13]\n - button \"Alert me\" [ref=e14]\n - button \"Ask my name\" [ref=e15]\n - search\n - textbox \"Search\" [ref=e16]\n - button \"Go\" [ref=e17]\n - list\n - listitem: One\n - listitem\n - text: Two\n - link \"anchor\" [ref=e18] → #t\n - table\n - row: Item | Price\n - row: Tea | 3\n - region \"Files\"\n - button \"Choose file\" [ref=e19]\n - button \"Send file\" [ref=e20]\n - iframe \"Same-origin frame\":\n - paragraph: Inside the frame\n - button \"Frame button\" [ref=e21]\n - text: Tall section\n - paragraph: The end of the page." + "ms": 7, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\n- banner\n - navigation \"Main\"\n - link \"Page two\" [ref=e1] → /page2\n - link \"Open help\" [ref=e2] → /help\n - link \"Download report\" [ref=e3] → /download\n- main\n - heading \"Order form\" [level=1] [ref=e4]\n - paragraph: Fill in the form below and press Greet.\n - textbox \"Name\" [ref=e5]: Ada\n - textbox \"Email\" [ref=e6]\n - textbox \"Password\" [ref=e7]\n - combobox \"Colour\" [ref=e8]: Green\n - option \"Red\"\n - option \"Green\" [selected]\n - option \"Blue\"\n - checkbox \"Subscribe\" [ref=e9]\n - group \"Size\"\n - radio \"Small\" [checked] [ref=e10]\n - radio \"Large\" [ref=e11]\n - textbox \"Notes\" [ref=e12]: hello\n - button \"Greet\" [ref=e13]\n - button \"Alert me\" [ref=e14]\n - button \"Ask my name\" [ref=e15]\n - search\n - textbox \"Search\" [ref=e16]\n - button \"Go\" [ref=e17]\n - list\n - listitem: One\n - listitem\n - text: Two\n - link \"anchor\" [ref=e18] → #t\n - table\n - row: Item | Price\n - row: Tea | 3\n - region \"Files\"\n - button \"Choose file\" [ref=e19]\n - button \"Send file\" [ref=e20]\n - iframe \"Same-origin frame\":\n - paragraph: Inside the frame\n - button \"Frame button\" [ref=e21]\n - text: Tall section\n - paragraph: The end of the page." }, { "name": "fill: text field, select by option text, checkbox", "ok": true, - "ms": 22, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nSet textbox \"Name\" (e5) to \"Bob\".\nTab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nSelected \"Blue\" in combobox \"Colour\" (e8).\nTab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nChecked checkbox \"Subscribe\" (e9); it is now checked.\nTab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\ncheckbox \"Subscribe\" (e9) was already checked." + "ms": 13, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nSet textbox \"Name\" (e5) to \"Bob\".\nTab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nSelected \"Blue\" in combobox \"Colour\" (e8).\nTab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nChecked checkbox \"Subscribe\" (e9); it is now checked.\nTab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\ncheckbox \"Subscribe\" (e9) was already checked." }, { "name": "click by ref in a background tab", "ok": true, - "ms": 618, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nClicked button \"Greet\" (e13)." + "ms": 620, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nClicked button \"Greet\" (e13)." }, { "name": "console: load-time and click messages, errorsOnly", "ok": true, "ms": 5, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\n3 console messages (oldest first; Stem attached at 18:19:15):\n18:19:15 log form loaded 42 (form:31)\n18:19:15 debug [recommendation] [DOM] Password field is not contained in a form: (More info: https://goo.gl/9p2vKq) %o (form)\n18:19:15 error greet clicked Bob (form:35)" + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\n3 console messages (oldest first; Stem attached at 21:05:06):\n21:05:06 log form loaded 42 (form:31)\n21:05:06 debug [recommendation] [DOM] Password field is not contained in a form: (More info: https://goo.gl/9p2vKq) %o (form)\n21:05:07 error greet clicked Bob (form:35)" }, { "name": "network: list and one request with its body", "ok": true, - "ms": 3, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\n1 requests matching \"/api/data\" since Stem attached at 18:19:15 (oldest first; `network` with `request`: an id for its headers and body):\nr5 GET 200 fetch 239 B http://127.0.0.1:58345/api/data?name=Bob\n---\nTab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nr5 GET http://127.0.0.1:58345/api/data?name=Bob\nStatus: 200 OK · type fetch · application/json · 239 B\nRequest headers:\n Accept-Language: en-US,en;q=0.9\n Referer: http://127.0.0.1:58345/form?tab=F\n User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/149.0.0.0 Safari/537.36\n sec-ch-ua: \"Chromium\";v=\"149\", \"Not)A;Brand\";v=\"24\"\n sec-ch-ua-mobile: ?0\n sec-ch-ua-platform: \"macOS\"\nResponse headers:\n Connection: keep-alive\n Date: Mon, 05 Oct 2026 16:19:15 GMT\n Keep-Alive: timeout=5\n Transfer-Encoding: chunked\n cache-control: no-store\n content-type: application/json\nResponse body:\n{\"greeting\":\"Hello Bob\",\"items\":[1,2,3]}" + "ms": 2, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\n1 requests matching \"/api/data\" since Stem attached at 21:05:06 (oldest first; `network` with `request`: an id for its headers and body):\nr5 GET 200 fetch 239 B http://127.0.0.1:49512/api/data?name=Bob\n---\nTab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nr5 GET http://127.0.0.1:49512/api/data?name=Bob\nStatus: 200 OK · type fetch · application/json · 239 B\nRequest headers:\n Accept-Language: en-US,en;q=0.9\n Referer: http://127.0.0.1:49512/form?tab=F\n User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/149.0.0.0 Safari/537.36\n sec-ch-ua: \"Chromium\";v=\"149\", \"Not)A;Brand\";v=\"24\"\n sec-ch-ua-mobile: ?0\n sec-ch-ua-platform: \"macOS\"\nResponse headers:\n Connection: keep-alive\n Date: Mon, 19 Oct 2026 19:05:07 GMT\n Keep-Alive: timeout=5\n Transfer-Encoding: chunked\n cache-control: no-store\n content-type: application/json\nResponse body:\n{\"greeting\":\"Hello Bob\",\"items\":[1,2,3]}" }, { "name": "type + submit navigates; wait for text", "ok": true, - "ms": 311, - "detail": "Tab 442773630 · Search results — http://127.0.0.1:58345/search?q=stem+rocks\nTyped 10 characters into textbox \"Search\" (e16) and pressed Enter. The tab went to http://127.0.0.1:58345/search?q=stem+rocks.\nTab 442773630 · Search results — http://127.0.0.1:58345/search?q=stem+rocks\nThe text \"Results for stem rocks\" appeared after 0.0 s." + "ms": 321, + "detail": "Tab 1293553522 · Search results — http://127.0.0.1:49512/search?q=stem+rocks\nTyped 10 characters into textbox \"Search\" (e16) and pressed Enter. The tab went to http://127.0.0.1:49512/search?q=stem+rocks.\nTab 1293553522 · Search results — http://127.0.0.1:49512/search?q=stem+rocks\nThe text \"Results for stem rocks\" appeared after 0.0 s." }, { "name": "navigate back, then wait for delayed text and a timeout", "ok": true, - "ms": 2784, - "detail": "Tab 442773630 · Delayed — http://127.0.0.1:58345/delayed\nThe text \"ready NOW\" appeared after 1.5 s.\nTab 442773630 · Delayed — http://127.0.0.1:58345/delayed\nThe text \"never there\" did not appear within 1 s.\nTab 442773630 · Delayed — http://127.0.0.1:58345/delayed\nA URL containing \"/delayed\" appeared after 0.0 s." + "ms": 2838, + "detail": "Tab 1293553522 · Delayed — http://127.0.0.1:49512/delayed\nThe text \"ready NOW\" appeared after 1.6 s.\nTab 1293553522 · Delayed — http://127.0.0.1:49512/delayed\nThe text \"never there\" did not appear within 1 s.\nTab 1293553522 · Delayed — http://127.0.0.1:49512/delayed\nA URL containing \"/delayed\" appeared after 0.0 s." }, { "name": "press: Meta+A then Backspace clears a field (Mac editing commands), Tab moves focus", "ok": true, "ms": 781, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nPressed Meta+A." + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nPressed Meta+A." }, { "name": "scroll: wheel down in a background tab, then a ref into view", "ok": true, - "ms": 462, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nScrolled the page down: it is now at y=330 of 2592 (the view is 413 px high).\nTab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nScrolled button \"Greet\" (e13) into view: its top is 196 px from the top of the 413 px high view." + "ms": 477, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nScrolled the page down: it is now at y=330 of 2592 (the view is 413 px high).\nTab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nScrolled button \"Greet\" (e13) into view: its top is 196 px from the top of the 413 px high view." }, { "name": "evaluate: value, top-level await, a thrown error", "ok": true, - "ms": 157, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\n\"Fixture form\"\nTab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\n{\n \"n\": 42\n}\nThe script threw: ReferenceError: nope is not defined at :1:1" + "ms": 243, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\n\"Fixture form\"\nTab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\n{\n \"n\": 42\n}\nThe script threw: ReferenceError: nope is not defined at :1:1" }, { "name": "dialog: an alert during a click does not hang; other actions refuse; prompt answered with text", "ok": true, - "ms": 670, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nClicked button \"Alert me\" (e14). The page opened a dialog: alert \"Hi there\". Answer it with `dialog` (accept: true or false) before doing anything else in this tab. (18 ms)" + "ms": 662, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nClicked button \"Alert me\" (e14). The page opened a dialog: alert \"Hi there\". Answer it with `dialog` (accept: true or false) before doing anything else in this tab. (10 ms)" }, { "name": "screenshot of a background tab: CSS-pixel size, not blank, marker hidden", "ok": true, - "ms": 189, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nScreenshot of the visible part of the page, 741×413; coordinates are CSS pixels of this picture. | view 756x413 | stdev 34.0 | edge 255,255,255 | Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nFull-page screenshot, 741×1652 (the top 1652 px of a 3039 px page). Its coordinates are page coordinates; `coordinate` clicks use the visible part of the page, so scroll first or use a ref." + "ms": 145, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nScreenshot of the visible part of the page, 741×413; coordinates are CSS pixels of this picture. | view 756x413 | stdev 34.0 | edge 255,255,255 | Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nFull-page screenshot, 741×1652 (the top 1652 px of a 3039 px page). Its coordinates are page coordinates; `coordinate` clicks use the visible part of the page, so scroll first or use a ref." }, { "name": "upload through a hidden file input; the server receives the bytes", "ok": true, - "ms": 343, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nAttached upload-me.txt (33 B) to the file input (from button \"Choose file\" (e19)). Many sites upload as soon as a file is chosen; if this one has a send or upload button, click it." + "ms": 338, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nAttached upload-me.txt (33 B) to the file input (from button \"Choose file\" (e19)). Many sites upload as soon as a file is chosen; if this one has a send or upload button, click it." }, { "name": "a click on an upload button is answered by a file chooser, not a native dialog", "ok": true, - "ms": 310, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F\nClicked button \"Choose file\" (e19). It opened a file chooser: answer it with `upload` (same ref, and the files)." + "ms": 313, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F\nClicked button \"Choose file\" (e19). It opened a file chooser: answer it with `upload` (same ref, and the files)." }, { "name": "download: reported on the next result exactly once, with its path", "ok": true, - "ms": 318, - "detail": "{\"path\":\"/var/folders/9g/7y5ws8zs42jckrc1h2363fw80000gn/T/stem-extension-live-wlUdpY/downloads/report.txt\",\"name\":\"report.txt\",\"size\":28,\"mime\":\"text/plain\"} | Downloads from this run:\n- report.txt — done, 28 B" + "ms": 316, + "detail": "{\"path\":\"/var/folders/9g/7y5ws8zs42jckrc1h2363fw80000gn/T/stem-extension-live-40Yo6h/downloads/report.txt\",\"name\":\"report.txt\",\"size\":28,\"mime\":\"text/plain\"} | Downloads from this run:\n- report.txt — done, 28 B" }, { "name": "a download the user makes during a run is never the run’s", "ok": true, - "ms": 618, + "ms": 617, "detail": "report (1).txt, report.txt on disk; run lists: Downloads from this run: | - report.txt — done, 28 B" }, { "name": "a target=_blank click opens a background tab; the user’s view does not switch", "ok": true, - "ms": 830, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F#stem-test-user-download\nClicked link \"Open help\" (e2).\nIt opened tab 442773631 in the background (yours to use or close): http://127.0.0.1:58345/help" + "ms": 835, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F#stem-test-user-download\nClicked link \"Open help\" (e2).\nIt opened tab 1293553523 in the background (yours to use or close): http://127.0.0.1:49512/help" }, { "name": "close: own tab closes, the user’s tab is refused", "ok": true, "ms": 8, - "detail": "Closed tab 442773631 (Help). | You can only close tabs you opened." + "detail": "Closed tab 1293553523 (Help). | You can only close tabs you opened." }, { "name": "refuses chrome:// and file: (open, and acting in a tab showing one)", "ok": true, - "ms": 1, + "ms": 6, "detail": "Stem can't open that: it shows a browser page (chrome://settings/), which extensions are not allowed to control (chrome: pages).\nStem can't open that: it is a file on this Mac, which Stem never opens — uploads go through `upload` with files Stem holds.\nStem can't open that: it is a file on this Mac, which Stem never opens — uploads go through `upload` with files Stem holds." }, { "name": "two runs at once in different tabs; a shared tab gets a warning; actions serialize", "ok": true, - "ms": 649, - "detail": "Note: another Stem run also worked in this tab in the last 2 minutes, so the page may have changed under you. | A: Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F#stem-test-user-download B: Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F#stem-test-user-download" + "ms": 721, + "detail": "Note: another Stem run also worked in this tab in the last 2 minutes, so the page may have changed under you. | A: Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F#stem-test-user-download B: Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F#stem-test-user-download" }, { "name": "minimized window: click and scroll still work in a background tab; screenshot works or says why", "ok": true, - "ms": 2376, - "detail": "click 308 ms | Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F#stem-test-minimize | screenshot ok 741x413 (56 ms) | visibilityState visible" + "ms": 2371, + "detail": "click 314 ms | Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F#stem-test-minimize | screenshot ok 741x413 (50 ms) | visibilityState visible" }, { "name": "marker: present, invisible to page script (no binding, closed shadow root)", @@ -167,14 +167,20 @@ { "name": "marker Stop button → `stopped` for every run using that tab; later requests refused", "ok": true, - "ms": 1518, + "ms": 1516, "detail": "stopped: thread-a, thread-b; marker removed" }, { "name": "end: runs forgotten; a new run re-attaches with fresh history", "ok": true, - "ms": 2009, - "detail": "Tab 442773630 · Fixture form — http://127.0.0.1:58345/form?tab=F#stem-test-press-stop\n3 console messages (oldest first; 3 from before Stem attached at 18:19:29):\n18:19:19 log form loaded 42 (form:31)\n18:19:26 error greet clicked (form:35)\n18:19:19 debug [recommendation] [DOM] Password field is not contained in a form: (More info: https://goo.gl/9p2vKq) %o (form)" + "ms": 2019, + "detail": "Tab 1293553522 · Fixture form — http://127.0.0.1:49512/form?tab=F#stem-test-press-stop\n3 console messages (oldest first; 3 from before Stem attached at 21:05:20):\n21:05:10 log form loaded 42 (form:31)\n21:05:17 error greet clicked (form:35)\n21:05:10 debug [recommendation] [DOM] Password field is not contained in a form: (More info: https://goo.gl/9p2vKq) %o (form)" + }, + { + "name": "another extension’s frame in the page is taken out so Stem can attach; one added later too", + "ok": true, + "ms": 993, + "detail": "Set textbox \"City\" (e2) to \"Most\"." }, { "name": "stale and unknown refs say so", @@ -185,100 +191,118 @@ ], "markerLog": [ { - "at": 1791217155147, - "href": "http://127.0.0.1:58345/user", + "at": 1792436706260, + "href": "http://127.0.0.1:49512/user", "present": false, - "doc": "6asn3v498kb" + "doc": "60gacdlevfi" }, { - "at": 1791217155599, - "href": "http://127.0.0.1:58345/form?tab=F", + "at": 1792436706941, + "href": "http://127.0.0.1:49512/form?tab=F", "present": false, - "doc": "nkmxikffj67" + "doc": "b4aotsyk856" }, { - "at": 1791217156403, - "href": "http://127.0.0.1:58345/search?q=stem+rocks", + "at": 1792436707723, + "href": "http://127.0.0.1:49512/search?q=stem+rocks", "present": false, - "doc": "4m6t2l9dceh" + "doc": "t6gc7zxkeps" }, { - "at": 1791217156692, - "href": "http://127.0.0.1:58345/form?tab=F", + "at": 1792436708033, + "href": "http://127.0.0.1:49512/form?tab=F", "present": false, - "doc": "tpxjqoy9u" + "doc": "w7e6rt9if2" + }, + { + "at": 1792436708157, + "href": "http://127.0.0.1:49512/delayed", + "present": false, + "doc": "tjw6xcvaej" + }, + { + "at": 1792436709296, + "href": "http://127.0.0.1:49512/delayed", + "present": true, + "doc": "tjw6xcvaej" }, { - "at": 1791217156813, - "href": "http://127.0.0.1:58345/delayed", + "at": 1792436710876, + "href": "http://127.0.0.1:49512/form?tab=F", "present": false, - "doc": "3lc4ba4bdaj" + "doc": "0bomtwkhthe" }, { - "at": 1791217157903, - "href": "http://127.0.0.1:58345/delayed", + "at": 1792436711323, + "href": "http://127.0.0.1:49512/form?tab=F", "present": true, - "doc": "3lc4ba4bdaj" + "doc": "0bomtwkhthe" + }, + { + "at": 1792436714787, + "href": "http://127.0.0.1:49512/help", + "present": false, + "doc": "vvvnji2t7rl" }, { - "at": 1791217159489, - "href": "http://127.0.0.1:58345/form?tab=F", + "at": 1792436715652, + "href": "http://127.0.0.1:49512/page2", "present": false, - "doc": "uye1ndq8vgl" + "doc": "1n8xuvk9biw" }, { - "at": 1791217160222, - "href": "http://127.0.0.1:58345/form?tab=F", + "at": 1792436716321, + "href": "http://127.0.0.1:49512/page2", "present": true, - "doc": "uye1ndq8vgl" + "doc": "1n8xuvk9biw" }, { - "at": 1791217163350, - "href": "http://127.0.0.1:58345/help", + "at": 1792436718711, + "href": "http://127.0.0.1:49512/form?tab=F#stem-test-press-stop", "present": false, - "doc": "yl7jw5t70n8" + "doc": "0bomtwkhthe" }, { - "at": 1791217164207, - "href": "http://127.0.0.1:58345/page2", + "at": 1792436718713, + "href": "http://127.0.0.1:49512/page2", "present": false, - "doc": "pcnwgbfstf" + "doc": "1n8xuvk9biw" }, { - "at": 1791217165216, - "href": "http://127.0.0.1:58345/page2", + "at": 1792436720731, + "href": "http://127.0.0.1:49512/form?tab=F#stem-test-press-stop", "present": true, - "doc": "pcnwgbfstf" + "doc": "0bomtwkhthe" }, { - "at": 1791217167203, - "href": "http://127.0.0.1:58345/form?tab=F#stem-test-press-stop", + "at": 1792436720734, + "href": "http://127.0.0.1:49512/form?tab=F#stem-test-press-stop", "present": false, - "doc": "uye1ndq8vgl" + "doc": "0bomtwkhthe" }, { - "at": 1791217167205, - "href": "http://127.0.0.1:58345/page2", + "at": 1792436722280, + "href": "http://127.0.0.1:49512/extframe", "present": false, - "doc": "pcnwgbfstf" + "doc": "2eydol4qk21" }, { - "at": 1791217169211, - "href": "http://127.0.0.1:58345/form?tab=F#stem-test-press-stop", + "at": 1792436722393, + "href": "http://127.0.0.1:49512/extframe", "present": true, - "doc": "uye1ndq8vgl" + "doc": "2eydol4qk21" }, { - "at": 1791217169214, - "href": "http://127.0.0.1:58345/form?tab=F#stem-test-press-stop", + "at": 1792436723230, + "href": "http://127.0.0.1:49512/extframe#stem-test-foreign-frame", "present": false, - "doc": "uye1ndq8vgl" + "doc": "2eydol4qk21" }, { - "at": 1791217170721, - "href": "http://127.0.0.1:58345/form?tab=F#stem-test-press-stop", + "at": 1792436723234, + "href": "http://127.0.0.1:49512/form?tab=F#stem-test-press-stop", "present": true, - "doc": "uye1ndq8vgl" + "doc": "0bomtwkhthe" } ] }, diff --git a/ios/Stem/Markdown/Components/DiagramView.swift b/ios/Stem/Markdown/Components/DiagramView.swift new file mode 100644 index 00000000..51b02ef1 --- /dev/null +++ b/ios/Stem/Markdown/Components/DiagramView.swift @@ -0,0 +1,215 @@ +import SwiftUI +import WebKit + +/// Draws Mermaid source the way the desktop's does: with pele, in a +/// web view, links and images off so a diagram can only be a picture, and the +/// SVG sanitized by DOMPurify before it is inserted, as on the desktop. +/// pele.global.js and purify.min.js ship in the app bundle (project.yml copies +/// them from the repo's node_modules, the versions the desktop renders with) and +/// are injected as user scripts with the draw function; the page itself runs no +/// script at all (its CSP allows none), has no base URL, may fetch nothing and +/// may not navigate anywhere. The web view reports the drawing's +/// height back and the cell sizes to it. +/// +/// While `live` (the reply is still streaming), each new source redraws in +/// place from its complete lines, keeping the last drawing that worked, so the +/// diagram grows as it is written. Once the source is final, one pele can't +/// parse reports an error and the caller shows the source instead. +struct DiagramView: View { + let source: String + var live = false + /// Called once if pele could not draw the final source. + var onError: () -> Void = {} + + @Environment(\.colorScheme) private var scheme + @State private var height: CGFloat = 0 + + var body: some View { + DiagramWebView(source: source, live: live, dark: scheme == .dark, height: $height, onError: onError) + // Keyed on the scheme, so a light/dark flip redraws in the new colors. + .id(scheme) + .frame(height: max(height, 60)) + .overlay { + if height == 0 { ProgressView().controlSize(.small) } + } + } +} + +private struct DiagramWebView: UIViewRepresentable { + let source: String + let live: Bool + let dark: Bool + @Binding var height: CGFloat + let onError: () -> Void + + func makeCoordinator() -> Coordinator { Coordinator(self) } + + func makeUIView(context: Context) -> WKWebView { + let config = WKWebViewConfiguration() + config.websiteDataStore = .nonPersistent() + // A weak hop: the controller retains its handlers, and the coordinator owns nothing of the view. + config.userContentController.add(WeakHandler(context.coordinator), name: "diagram") + // User scripts are not subject to the page's CSP, so these run while the page itself may run nothing. + for script in [Self.peleScript, Self.purifyScript].compactMap({ $0 }) { + config.userContentController.addUserScript( + WKUserScript(source: script, injectionTime: .atDocumentStart, forMainFrameOnly: true) + ) + } + config.userContentController.addUserScript( + WKUserScript(source: Self.drawScript, injectionTime: .atDocumentEnd, forMainFrameOnly: true) + ) + let view = WKWebView(frame: .zero, configuration: config) + view.isOpaque = false + view.backgroundColor = .clear + view.scrollView.isScrollEnabled = false + view.scrollView.backgroundColor = .clear + view.navigationDelegate = context.coordinator + context.coordinator.drawn = (source, live) + // The first drawing comes from didFinish, once the draw function is in place. + view.loadHTMLString(Self.page(dark: dark), baseURL: nil) + return view + } + + func updateUIView(_ view: WKWebView, context: Context) { + let coordinator = context.coordinator + coordinator.parent = self + // A streaming source grew (or settled): redraw in place, no reload. + guard coordinator.drawn != (source, live) else { return } + coordinator.drawn = (source, live) + // Before the page has loaded, didFinish draws the latest source instead. + if coordinator.ready { coordinator.redraw(view) } + } + + static func dismantleUIView(_ view: WKWebView, coordinator: Coordinator) { + view.configuration.userContentController.removeScriptMessageHandler(forName: "diagram") + } + + final class Coordinator: NSObject, WKScriptMessageHandler, WKNavigationDelegate { + var parent: DiagramWebView + /// The source and mode the page was last asked to draw. + var drawn: (String, Bool) = ("", false) + /// True once the page (and its draw function) has loaded. + var ready = false + private var failed = false + private var loaded = false + init(_ parent: DiagramWebView) { self.parent = parent } + + func userContentController(_ controller: WKUserContentController, didReceive message: WKScriptMessage) { + guard let body = message.body as? [String: Any] else { return } + if let h = body["height"] as? Double, h > 0 { + parent.height = CGFloat(h) + } else if body["error"] != nil, !failed { + failed = true + parent.onError() + } + } + + func redraw(_ view: WKWebView) { + view.evaluateJavaScript("draw(\(DiagramWebView.json(drawn.0)), \(drawn.1))") + } + + func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) { + ready = true + redraw(webView) + } + + // The one page we load, and nothing after it: a diagram never navigates anywhere. + func webView( + _ webView: WKWebView, + decidePolicyFor action: WKNavigationAction, + decisionHandler: @escaping (WKNavigationActionPolicy) -> Void + ) { + if !loaded, action.targetFrame?.isMainFrame == true, action.request.url?.absoluteString == "about:blank" { + loaded = true + decisionHandler(.allow) + } else { + decisionHandler(.cancel) + } + } + } + + private final class WeakHandler: NSObject, WKScriptMessageHandler { + weak var target: WKScriptMessageHandler? + init(_ target: WKScriptMessageHandler) { self.target = target } + func userContentController(_ controller: WKUserContentController, didReceive message: WKScriptMessage) { + target?.userContentController(controller, didReceive: message) + } + } + + /// The colors the desktop draws with (Stem's light and dark tokens), as pele's CSS variables. + private static func themeVariables(dark: Bool) -> String { + let vars: [(String, String)] = [ + ("bg", dark ? "#242119" : "#faf8f3"), + ("fg", dark ? "#f0ece4" : "#23211d"), + ("muted", dark ? "#9b948a" : "#6d675d"), + ("line", dark ? "#9b948a" : "#6d675d"), + ("surface", dark ? "#2e2a23" : "#fffdf9"), + ("surface-alt", dark ? "#2f2b25" : "#efeae0"), + ("border", dark ? "#9b948a" : "#6d675d"), + ("accent", dark ? "#c79257" : "#9a6230"), + ("critical", "#c53030"), + ("font", "-apple-system, system-ui, sans-serif"), + ("series-1", dark ? "#3987e5" : "#2a78d6"), ("series-2", dark ? "#d95926" : "#eb6834"), + ("series-3", dark ? "#199e70" : "#1baf7a"), ("series-4", dark ? "#c98500" : "#eda100"), + ("series-5", dark ? "#d55181" : "#e87ba4"), ("series-6", "#008300"), + ("series-7", dark ? "#9085e9" : "#4a3aa7"), ("series-8", dark ? "#e66767" : "#e34948"), + ] + return vars.map { "--pele-\($0.0): \($0.1);" }.joined(separator: " ") + } + + /// pele.global.js and DOMPurify from the bundle, read once. + private static let peleScript = bundledScript("pele.global") + private static let purifyScript = bundledScript("purify.min") + + private static func bundledScript(_ name: String) -> String? { + Bundle.main.url(forResource: name, withExtension: "js").flatMap { try? String(contentsOf: $0, encoding: .utf8) } + } + + /// Draws into #d; the coordinator calls draw(source, live) for every source. + /// Streaming: only complete lines, and a line that doesn't parse yet keeps the last drawing. + private static let drawScript = """ + const post = (m) => window.webkit.messageHandlers.diagram.postMessage(m); + const box = document.getElementById('d'); + const report = () => post({ height: Math.ceil(box.getBoundingClientRect().height) }); + new ResizeObserver(report).observe(box); + function draw(source, live) { + const cut = source.lastIndexOf('\\n'); + const code = (live ? (cut === -1 ? '' : source.slice(0, cut)) : source).trim(); + if (!code) return; + try { + const { svg } = pele.render(code, { + idPrefix: 'stem-diagram', maxWidth: box.clientWidth || undefined, + fontFamily: '-apple-system, system-ui, sans-serif', fontSize: 14, + links: false, images: false + }); + // Pele escapes labels itself; the source is model text, so DOMPurify too. + box.innerHTML = DOMPurify.sanitize(svg, { USE_PROFILES: { svg: true, svgFilters: true } }); + report(); + } catch (e) { + if (!live) post({ error: String(e && e.message || e) }); + } + } + """ + + fileprivate static func json(_ value: Any) -> String { + guard let data = try? JSONSerialization.data(withJSONObject: value, options: [.fragmentsAllowed]), + let text = String(data: data, encoding: .utf8) else { return "null" } + // Inside a - - """ - } -} diff --git a/ios/Stem/Markdown/MdxGallery.swift b/ios/Stem/Markdown/MdxGallery.swift index fe7230b3..e06436b7 100644 --- a/ios/Stem/Markdown/MdxGallery.swift +++ b/ios/Stem/Markdown/MdxGallery.swift @@ -3,7 +3,8 @@ import SwiftUI /// Every MDX fixture (tests/fixtures/mdx) drawn natively, for previews and /// for simulator screenshots: launch with `-mdxGallery` to open it instead of -/// the app (add `-mdxOnly ` for one sample). Interactive components get a submit that only logs, never sends. +/// the app (add `-mdxOnly ` for one sample, and `-mdxStream` to type it +/// out the way a streaming reply arrives). Interactive components get a submit that only logs, never sends. struct MdxGallery: View { static let samples: [(String, String)] = [ ("callout", ##""" @@ -110,6 +111,22 @@ flowchart LR App --> Gateway --> Orders --> DB[(Postgres)] ``` +"""##), + // Not a fixture: a diagram long enough to watch grow under `-mdxStream`. + ("diagram-stream", ##""" + +```mermaid +sequenceDiagram + participant P as Phone + participant S as Stem server + participant R as Relay + P->>S: Pair with code + S-->>P: Device token + P->>S: Open chat + S->>R: Push subscription + R-->>P: Notification +``` + """##), ("form", ##""" I need a couple of things before planning. @@ -246,6 +263,8 @@ sudo apt install ./stem.deb ] @State private var lastSubmit: String? + /// Characters shown so far under `-mdxStream`; nil shows every sample whole. + @State private var typed: Int? @State private var formHost = MdxFormHost(place: .below) var body: some View { @@ -257,7 +276,7 @@ sudo apt install ./stem.deb ForEach(Array(Self.samples.filter { only == nil || $0.0 == only }.enumerated()), id: \.element.0) { i, sample in VStack(alignment: .leading, spacing: 8) { Text(sample.0).font(.caption.monospaced()).foregroundStyle(.tertiary) - MarkdownView(sample.1) + MarkdownView(typed.map { String(sample.1.prefix($0)) } ?? sample.1) .environment(\.mdxSource, MdxSource(key: "gallery:\(sample.0)", scrollId: sample.0, order: Double(i))) } .id(sample.0) @@ -273,6 +292,15 @@ sudo apt install ./stem.deb .safeAreaInset(edge: .bottom, spacing: 0) { MdxAnswerBar(host: formHost, running: false) } } .environment(\.mdxFormHost, formHost) + .task { + guard ProcessInfo.processInfo.arguments.contains("-mdxStream"), + let only = UserDefaults.standard.string(forKey: "mdxOnly"), + let sample = Self.samples.first(where: { $0.0 == only })?.1 else { return } + for i in 0...sample.count { + typed = i + try? await Task.sleep(for: .milliseconds(40)) + } + } .onAppear { formHost.ask = { lastSubmit = "About “\($0)”: " } } .environment(\.mdxActions, MdxActions(submit: { lastSubmit = $0; print("MDX submit:", $0) }, running: false)) .environment(\.mdxIsLatest, true) diff --git a/ios/Stem/Markdown/MdxView.swift b/ios/Stem/Markdown/MdxView.swift index ca1a2ff6..cf945163 100644 --- a/ios/Stem/Markdown/MdxView.swift +++ b/ios/Stem/Markdown/MdxView.swift @@ -38,6 +38,11 @@ struct MdxBlocksView: View { struct MdxComponentView: View { let component: MdxComponent + /// The data, or while the component is still streaming, its rows complete so far. + private var rows: String? { + component.closed ? component.data?.value : MdxDataParse.completeRows(component.data?.value) + } + var body: some View { let c = component switch c.name { @@ -45,13 +50,14 @@ struct MdxComponentView: View { case "Steps": MdxSteps(children: c.children) case "Tabs": MdxTabs(children: c.children) case "Collapsible": MdxCollapsible(title: c.attrs["title"], children: c.children) + // Still streaming: drawn from the rows complete so far. case "Chart": MdxChartView(type: c.attrs["type"], title: c.attrs["title"], unit: c.attrs["unit"], - data: c.data?.value, closed: c.closed) - case "DataTable": MdxDataTable(caption: c.attrs["caption"], data: c.data?.value, closed: c.closed) - case "Stats": MdxStatsView(data: c.data?.value, closed: c.closed) - case "Compare": MdxCompareView(recommend: c.attrs["recommend"], data: c.data?.value, closed: c.closed) - case "Diagram": MdxDiagram(title: c.attrs["title"], source: c.data?.value ?? "") + data: rows, closed: c.closed) + case "DataTable": MdxDataTable(caption: c.attrs["caption"], data: rows, closed: c.closed) + case "Stats": MdxStatsView(data: rows, closed: c.closed) + case "Compare": MdxCompareView(recommend: c.attrs["recommend"], data: rows, closed: c.closed) + case "Diagram": MdxDiagram(title: c.attrs["title"], source: c.data?.value ?? "", closed: c.closed) case "Quiz": MdxQuiz(topic: c.attrs["topic"], children: c.children) case "Form": MdxForm(prompt: c.attrs["prompt"], submitLabel: c.attrs["submitLabel"], children: c.children) case "Replies": if c.closed { MdxReplies(children: c.children) } diff --git a/ios/Stem/Models/Inbox.swift b/ios/Stem/Models/Inbox.swift index 16661f76..0bc8d588 100644 --- a/ios/Stem/Models/Inbox.swift +++ b/ios/Stem/Models/Inbox.swift @@ -44,7 +44,7 @@ enum MailSections { case .snoozed: out[.snoozed]!.append(c) case .archived: out[.archived]!.append(c) case .inbox: - if c.userUpdatedAt > (c.userSentAt ?? 0) || c.status == "aborted" { out[.inbox]!.append(c) } + if c.waitingOnUser || c.status == "aborted" { out[.inbox]!.append(c) } } if lastSent[c.id] != nil { out[.sent]!.append(c) } } @@ -161,7 +161,7 @@ extension MailSections { mail.conversations .filter { c in c.status == "working" - && !(c.userUpdatedAt > (c.userSentAt ?? 0)) + && !c.waitingOnUser && Inbox.placement(id: c.id, updatedAt: c.userUpdatedAt, state: mail.inbox, now: now) == .inbox } .sorted { ($0.userSentAt ?? 0) > ($1.userSentAt ?? 0) } diff --git a/ios/Stem/Models/Share.swift b/ios/Stem/Models/Share.swift new file mode 100644 index 00000000..c4deaba3 --- /dev/null +++ b/ios/Stem/Models/Share.swift @@ -0,0 +1,75 @@ +import Foundation + +// Public share links (docs/public-share-design.md). Mirrors src/shared/share.ts; +// the server builds, encrypts and uploads, the phone only asks and shows. + +enum ShareSource: Hashable { + case chat(threadId: String) + case mail(conversationId: String) + + var json: JSONValue { + switch self { + case .chat(let id): return .object(["kind": .string("chat"), "threadId": .string(id)]) + case .mail(let id): return .object(["kind": .string("mail"), "conversationId": .string(id)]) + } + } + var isMail: Bool { if case .mail = self { return true }; return false } +} + +/// What the user picked Share on: the source and the answer (turn or mail item id). +struct ShareRequest: Identifiable, Hashable { + let source: ShareSource + let anchorId: String + var id: String { anchorId } +} + +enum ShareScope: String, CaseIterable, Identifiable { + case thread, pair, answer + var id: String { rawValue } +} + +struct ShareImage: Decodable, Hashable { var mime: String; var dataUrl: String; var alt: String? } + +struct ShareMessage: Decodable, Hashable { + var role: String + var author: String? + var text: String + var model: String? + var images: [ShareImage]? +} + +struct ShareSnapshot: Decodable, Hashable { + var kind: String + var title: String + var messages: [ShareMessage] +} + +struct ShareCard: Codable, Hashable { var title: String; var description: String } + +struct ShareDraftPreview: Decodable, Hashable { + var snapshot: ShareSnapshot + var bytes: Int + var omittedAttachments: Int + /// Absent from servers before link previews. + var card: ShareCard? +} + +struct SharedLink: Decodable, Hashable, Identifiable { + var id: String + var url: String + var title: String + var expiresAt: Double +} + +struct ShareStatus: Decodable, Hashable { + var available: Bool + var code: String? + var message: String? + var address: String? +} + +enum ShareLimits { + static let ttlChoices = [30, 7, 1] + static let cardTitleMax = 120 + static let cardDescMax = 300 +} diff --git a/ios/Stem/Models/Types.swift b/ios/Stem/Models/Types.swift index 2088719c..5dbc8cdb 100644 --- a/ios/Stem/Models/Types.swift +++ b/ios/Stem/Models/Types.swift @@ -5,16 +5,28 @@ import Foundation // MARK: Chats -struct ChatListResult: Decodable { +struct ChatListResult: Codable { var chats: [ChatSummary] + /// Absent only from a server older than folders. + var folders: [ChatFolder]? var inbox: InboxState } -struct ChatSummary: Decodable, Identifiable, Hashable { +/// A user-made chat folder; `parentId` nil = top level. +struct ChatFolder: Codable, Identifiable, Hashable { + var id: String + var name: String + var parentId: String? + /// Sort order among siblings. + var order: Double +} + +struct ChatSummary: Codable, Identifiable, Hashable { var threadId: String var title: String var subject: String? var preview: String? + var folderId: String? var `private`: Bool? /// "md" when the chat runs as plain Markdown; nil means MDX. var format: String? @@ -210,11 +222,20 @@ struct MailConversation: Decodable, Hashable, Identifiable { var subject: String var participants: [String] var `private`: Bool? + /// Inter-Stem: a conversation with another person (only presence matters here). + var external: JSONValue? + /// Owned by a server feature ("memory-check") instead of a persona. + var owner: String? var status: String var updatedAt: Double var userUpdatedAt: Double var userSentAt: Double? + /// The user's latest approval answer; absent from older servers. + var userAnsweredAt: Double? var createdAt: Double + + /// Mail for the user landed after their last send or approval answer. + var waitingOnUser: Bool { userUpdatedAt > max(userSentAt ?? 0, userAnsweredAt ?? 0) } } struct MailItem: Decodable, Hashable, Identifiable { @@ -228,6 +249,10 @@ struct MailItem: Decodable, Hashable, Identifiable { var stale: Bool? /// Persona↔persona mail that reached a persona after it had answered: filed, no turn started. var filed: Bool? + /// A progress question asked mid-work, answered on the spot instead of delivered. + var statusQuery: Bool? + /// The on-the-spot answer to a progress question, written from the thread and the live work log. + var statusReply: Bool? var attachments: [MessageAttachment]? var images: [GeneratedImageRef]? var agentReplies: [String]? diff --git a/ios/Stem/Stores/AppModel.swift b/ios/Stem/Stores/AppModel.swift index fc66b08f..0f0913d7 100644 --- a/ios/Stem/Stores/AppModel.swift +++ b/ios/Stem/Stores/AppModel.swift @@ -7,7 +7,9 @@ enum AppTab: Hashable { case mail, chats, settings } /// Where a notification tap or deep link wants to go. enum Route: Hashable { case thread(String) - case newChat + /// A draft filed into the folder once it becomes a chat; nil = top level. + case newChat(folderId: String?) + case folder(String) case mail(String) } @@ -58,6 +60,7 @@ final class AppModel { session = nil CredentialStore.clear() DraftStore.clearAll() + ChatListStore.clearCache() chatPath = [] mailPath = [] } @@ -72,7 +75,7 @@ final class AppModel { case .mail: tab = .mail mailPath = [route] - case .thread, .newChat: + case .thread, .newChat, .folder: tab = .chats chatPath = [route] } diff --git a/ios/Stem/Stores/ChatListStore.swift b/ios/Stem/Stores/ChatListStore.swift index a796664a..1717b3a5 100644 --- a/ios/Stem/Stores/ChatListStore.swift +++ b/ios/Stem/Stores/ChatListStore.swift @@ -4,6 +4,7 @@ import Observation @MainActor @Observable final class ChatListStore { private(set) var chats: [ChatSummary] = [] + private(set) var folders: [ChatFolder] = [] private(set) var inbox = InboxState() private(set) var loaded = false var error: String? @@ -12,7 +13,37 @@ final class ChatListStore { @ObservationIgnored private weak var connection: Connection? @ObservationIgnored private var debounce: Task? - init(client: StemClient) { self.client = client } + init(client: StemClient) { + self.client = client + // The last list this pairing saw, so a cold start shows chats at once + // instead of a spinner while a few hundred KB come over a phone link. + if let data = try? Data(contentsOf: Self.cacheURL(client.creds.deviceId)), + let cached = try? JSONDecoder().decode(ChatListResult.self, from: data) { + apply(cached) + loaded = true + } + } + + private static var cacheDir: URL { + FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent("chat-list", isDirectory: true) + } + + private static func cacheURL(_ deviceId: String) -> URL { + cacheDir.appendingPathComponent("\(deviceId).json") + } + + /// Unpairing forgets the list along with the pairing. + static func clearCache() { try? FileManager.default.removeItem(at: cacheDir) } + + private func save(_ r: ChatListResult) { + let url = Self.cacheURL(client.creds.deviceId) + Task.detached(priority: .utility) { + guard let data = try? JSONEncoder().encode(r) else { return } + try? FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try? data.write(to: url, options: .atomic) + } + } func attach(_ c: Connection) { connection = c @@ -40,7 +71,9 @@ final class ChatListStore { func reload() async { do { - apply(try await client.call("chats:list", as: ChatListResult.self)) + let r = try await client.call("chats:list", as: ChatListResult.self) + apply(r) + save(r) error = nil } catch { self.error = error.localizedDescription @@ -50,9 +83,76 @@ final class ChatListStore { private func apply(_ r: ChatListResult) { chats = r.chats.sorted { Inbox.toMs($0.updatedAt) > Inbox.toMs($1.updatedAt) } + folders = r.folders ?? [] inbox = r.inbox } + // MARK: Folders + + func childFolders(of parentId: String?) -> [ChatFolder] { + folders.filter { $0.parentId == parentId }.sorted { $0.order < $1.order } + } + + func folder(_ id: String?) -> ChatFolder? { + id.flatMap { id in folders.first { $0.id == id } } + } + + /// The folder and every folder under it. + func subtree(_ id: String) -> Set { + var out: Set = [id] + var frontier = [id] + while let next = frontier.popLast() { + for f in folders where f.parentId == next && !out.contains(f.id) { + out.insert(f.id) + frontier.append(f.id) + } + } + return out + } + + /// Unread chats anywhere under the folder, so a bold row can't hide inside it. + func unreadCount(in id: String) -> Int { + let ids = subtree(id) + return chats.filter { c in c.folderId.map(ids.contains) == true && !isArchived(c) && isUnread(c) }.count + } + + /// Every folder in tree order with its depth, for the Move picker. + var folderTree: [(folder: ChatFolder, depth: Int)] { + var out: [(ChatFolder, Int)] = [] + func walk(_ parent: String?, _ depth: Int) { + for f in childFolders(of: parent) { + out.append((f, depth)) + walk(f.id, depth + 1) + } + } + walk(nil, 0) + return out + } + + func moveChats(_ ids: [String], to folderId: String?) async { + for id in ids { + if let i = chats.firstIndex(where: { $0.threadId == id }) { chats[i].folderId = folderId } + } + for id in ids { await mutate("chats:setFolder", [.string(id), folderId.map(JSONValue.string) ?? .null]) } + } + + /// Answers the new folder's id, told apart from the folders that were there before. + @discardableResult + func createFolder(_ name: String, in parentId: String?) async -> String? { + let before = Set(folders.map(\.id)) + await mutate("folders:create", [.string(name), parentId.map(JSONValue.string) ?? .null]) + return folders.first { !before.contains($0.id) }?.id + } + + func renameFolder(_ id: String, _ name: String) async { + await mutate("folders:rename", [.string(id), .string(name)]) + } + + /// Its chats and subfolders move up to its parent; nothing is deleted with it. + func deleteFolder(_ id: String) async { + await mutate("folders:delete", [.string(id)]) + } + func isUnread(_ c: ChatSummary) -> Bool { Inbox.isUnread(id: c.threadId, updatedAt: c.updatedAt, state: inbox, turnRunning: connection?.liveTurns[c.threadId] != nil) @@ -77,7 +177,11 @@ final class ChatListStore { } private func mutate(_ channel: String, _ args: [JSONValue]) async { - do { apply(try await client.call(channel, args, as: ChatListResult.self)) } + do { + let r = try await client.call(channel, args, as: ChatListResult.self) + apply(r) + save(r) + } catch { self.error = error.localizedDescription } } } diff --git a/ios/Stem/Stores/DraftStore.swift b/ios/Stem/Stores/DraftStore.swift index 8c2d6856..9d93ea23 100644 --- a/ios/Stem/Stores/DraftStore.swift +++ b/ios/Stem/Stores/DraftStore.swift @@ -91,7 +91,7 @@ enum Uploads { for a in items { guard var data = a.data else { throw StemError(message: "\(a.name) is no longer on this phone") } var name = a.name, mime = a.mime - if mime == "image/heic" || mime == "image/heif", let img = UIImage(data: data), let jpg = img.jpegData(compressionQuality: 0.85) { + if isHeic(name: name, mime: mime), let img = UIImage(data: data), let jpg = img.jpegData(compressionQuality: 0.85) { data = jpg mime = "image/jpeg" name = (name as NSString).deletingPathExtension + ".jpg" @@ -101,16 +101,25 @@ enum Uploads { return out } + /// HEIC/HEIF stills and sequences, by MIME or extension. Windows and Linux + /// clients have no HEVC decoder, so these never leave the phone as-is. + static func isHeic(name: String, mime: String) -> Bool { + let m = mime.lowercased() + if m.hasPrefix("image/heic") || m.hasPrefix("image/heif") { return true } + return ["heic", "heif", "hif"].contains((name as NSString).pathExtension.lowercased()) + } + /// Note images travel inline: the memory model reads the bytes directly. static func inline(_ items: [DraftAttachment]) -> [TurnAttachment] { items.compactMap { a in guard let d = a.data else { return nil } - var data = d, mime = a.mime - if let img = UIImage(data: d), d.count > 1_500_000 || mime.contains("heic") || mime.contains("heif"), + var data = d, mime = a.mime, name = a.name + if let img = UIImage(data: d), d.count > 1_500_000 || isHeic(name: a.name, mime: mime), let jpg = img.jpegData(compressionQuality: 0.8) { data = jpg; mime = "image/jpeg" + if isHeic(name: name, mime: a.mime) { name = (name as NSString).deletingPathExtension + ".jpg" } } - return TurnAttachment(name: a.name, path: nil, dataBase64: data.base64EncodedString(), mime: mime) + return TurnAttachment(name: name, path: nil, dataBase64: data.base64EncodedString(), mime: mime) } } } diff --git a/ios/Stem/Stores/MailStore.swift b/ios/Stem/Stores/MailStore.swift index e396cb7b..20480617 100644 --- a/ios/Stem/Stores/MailStore.swift +++ b/ios/Stem/Stores/MailStore.swift @@ -7,8 +7,15 @@ final class MailStore { private(set) var sections: [MailFolder: [MailConversation]] = [:] private(set) var loaded = false var error: String? + /// Whole mails of the conversations on screen. The list itself carries only + /// what its rows show (mail:summary); a conversation's mail comes on open. + private(set) var threads: [String: [MailItem]] = [:] private let client: StemClient + /// Conversations on screen, with how many views show each. + @ObservationIgnored private var openCounts: [String: Int] = [:] + /// False on a server older than mail:summary: the whole list it sends holds every mail already. + @ObservationIgnored private var slim = true init(client: StemClient) { self.client = client } @@ -24,12 +31,46 @@ final class MailStore { func reload() async { do { - apply(try await client.call("mail:list", as: MailListResult.self)) + apply(try await fetchList()) error = nil } catch { self.error = error.localizedDescription } loaded = true + await withTaskGroup(of: Void.self) { group in + for id in openCounts.keys { group.addTask { await self.fetchThread(id) } } + } + } + + private func fetchList() async throws -> MailListResult { + if slim { + do { return try await client.call("mail:summary", as: MailListResult.self) } + catch let e as StemError where e.message.contains("no handler registered") { slim = false } + } + return try await client.call("mail:list", as: MailListResult.self) + } + + private struct OpenResult: Decodable { var items: [MailItem] } + + private func fetchThread(_ id: String) async { + guard slim else { return } + do { + threads[id] = try await client.call("mail:open", [.string(id)], as: OpenResult.self).items + } catch { + self.error = error.localizedDescription + } + } + + /// A conversation came on screen: fetch its mail, and keep it fresh while it stays. + func open(_ id: String) async { + openCounts[id, default: 0] += 1 + await fetchThread(id) + } + + func close(_ id: String) { + let n = (openCounts[id] ?? 1) - 1 + openCounts[id] = n > 0 ? n : nil + if n <= 0 { threads[id] = nil } } private func apply(_ r: MailListResult) { @@ -43,22 +84,28 @@ final class MailStore { func conversation(_ id: String) -> MailConversation? { mail.conversations.first { $0.id == id } } + /// A conversation's mails, oldest first. Empty for the moment between opening + /// it and its mail arriving: the list's copies are cut-down previews. func items(_ id: String) -> [MailItem] { - mail.items.filter { $0.conversationId == id }.sorted { $0.at < $1.at } + let all = slim ? threads[id] ?? [] : mail.items.filter { $0.conversationId == id } + return all.sorted { $0.at < $1.at } } - // Every mutator answers with the fresh list. + // Mutators: the server answers each with the whole mail list; the phone + // asks it to leave that out and re-reads the slim one instead. + /// Starts a conversation; answers with the list it now appears in. func compose(_ input: MailComposeInput) async throws -> MailListResult { - let r = try await client.call("mail:compose", [.from(input)], as: MailListResult.self) - apply(r) - return r + try await client.run("mail:compose", [.from(input)]) + await reload() + return mail } func reply(_ id: String, body: String, attachments: [TurnAttachment]) async throws { var args: [JSONValue] = [.string(id), .string(body)] if !attachments.isEmpty { args.append(.from(attachments)) } - apply(try await client.call("mail:reply", args, as: MailListResult.self)) + try await client.run("mail:reply", args) + await reload() } func addParticipant(_ id: String, persona: String) async { @@ -85,7 +132,8 @@ final class MailStore { } private func mutate(_ channel: String, _ args: [JSONValue]) async { - do { apply(try await client.call(channel, args, as: MailListResult.self)) } + do { try await client.run(channel, args) } catch { self.error = error.localizedDescription } + await reload() } } diff --git a/ios/Stem/Stores/ThreadStore.swift b/ios/Stem/Stores/ThreadStore.swift index 35c80d25..4b868c64 100644 --- a/ios/Stem/Stores/ThreadStore.swift +++ b/ios/Stem/Stores/ThreadStore.swift @@ -28,6 +28,8 @@ final class ThreadStore { /// The open chat's own format ("mdx" or "md"), stored with it on the server. /// A draft has none yet: it starts in the composer's default. private(set) var format: String? + /// Where a draft is filed once its first send makes it a chat. + let folderId: String? var error: String? let session: Session @@ -40,9 +42,10 @@ final class ThreadStore { /// turn it already started instead of a duplicate chat. @ObservationIgnored private var unanswered: (text: String, files: Int, turnId: String)? - init(session: Session, threadId: String?) { + init(session: Session, threadId: String?, folderId: String? = nil) { self.session = session self.threadId = threadId + self.folderId = folderId if let id = threadId, let row = session.chats.chats.first(where: { $0.threadId == id }) { isPrivate = row.private == true format = row.format ?? "mdx" @@ -369,6 +372,7 @@ final class ThreadStore { threadId = t self.isPrivate = isPrivate format = s.format + if let folderId { Task { await session.chats.moveChats([t], to: folderId) } } } pendingTurnId = nil if activityLabel == "Sending" { activityLabel = nil } diff --git a/ios/Stem/Transport/StemClient.swift b/ios/Stem/Transport/StemClient.swift index 7c38d211..aa9f46c9 100644 --- a/ios/Stem/Transport/StemClient.swift +++ b/ios/Stem/Transport/StemClient.swift @@ -29,8 +29,14 @@ struct StemClient: Sendable { /// Calls a server channel. Args are positional, like the desktop preload. func call(_ channel: String, _ args: [JSONValue] = [], as: T.Type = T.self) async throws -> T { + try await call(channel, args, as: T.self, discardResult: false) + } + + private func call(_ channel: String, _ args: [JSONValue], as: T.Type, discardResult: Bool) async throws -> T { var req = URLRequest(url: baseURL.appendingPathComponent("rpc")) req.httpMethod = "POST" + // The server leaves the answer out (older servers send it anyway). + if discardResult { req.setValue("none", forHTTPHeaderField: "X-Stem-Result") } req.setValue("Bearer \(creds.token)", forHTTPHeaderField: "Authorization") req.setValue("application/json", forHTTPHeaderField: "Content-Type") req.httpBody = try JSONEncoder().encode(["channel": JSONValue.string(channel), "args": .array(args)]) @@ -53,7 +59,7 @@ struct StemClient: Sendable { /// Fire-and-check for channels whose answer the caller ignores. func run(_ channel: String, _ args: [JSONValue] = []) async throws { - _ = try await call(channel, args, as: Empty.self) + _ = try await call(channel, args, as: Empty.self, discardResult: true) } struct UploadResult: Decodable { let handle: String; let name: String; let size: Int? } diff --git a/ios/Stem/Views/ChatListView.swift b/ios/Stem/Views/ChatListView.swift index e37d02be..8b62bd31 100644 --- a/ios/Stem/Views/ChatListView.swift +++ b/ios/Stem/Views/ChatListView.swift @@ -3,93 +3,322 @@ import SwiftUI struct ChatListView: View { @Environment(AppModel.self) private var app @Environment(Session.self) private var session - @State private var query = "" - @State private var showArchived = false var body: some View { @Bindable var app = app NavigationStack(path: $app.chatPath) { - List { - if !session.approvals.queue.isEmpty { - Section("Waiting for you") { - ForEach(session.approvals.queue) { a in - ApprovalCard(approval: a, showThreadLink: true) - .listRowInsets(EdgeInsets(top: 6, leading: 12, bottom: 6, trailing: 12)) - } + ChatFolderList(folderId: nil) + .navigationDestination(for: Route.self) { route in + switch route { + case .thread(let id): ThreadView(session: session, threadId: id).id(id) + case .newChat(let folderId): ThreadView(session: session, threadId: nil, folderId: folderId) + case .folder(let id): ChatFolderView(folderId: id) + case .mail(let id): MailConversationView(conversationId: id) } } - Section { - ForEach(rows) { chat in - NavigationLink(value: Route.thread(chat.threadId)) { ChatRow(chat: chat) } - .swipeActions(edge: .leading) { - let unread = session.chats.isUnread(chat) - Button { Task { await session.chats.setRead([chat.threadId], unread) } } label: { - Label(unread ? "Read" : "Unread", systemImage: unread ? "envelope.open" : "envelope.badge") - } - .tint(.blue) - } - .swipeActions(edge: .trailing) { - let archived = session.chats.isArchived(chat) - Button { Task { await session.chats.setArchived([chat.threadId], !archived) } } label: { - Label(archived ? "Unarchive" : "Archive", systemImage: "archivebox") - } - .tint(.gray) - Button(role: .destructive) { Task { await session.chats.delete(chat.threadId) } } label: { - Label("Delete", systemImage: "trash") - } - } + .task { if !session.chats.loaded { await session.chats.reload() } } + } + } +} + +/// One folder opened from the list: its subfolders, then its chats. +struct ChatFolderView: View { + let folderId: String + var body: some View { ChatFolderList(folderId: folderId) } +} + +/// The chat list at one level of the folder tree (nil = top level), the way +/// the desktop sidebar's Chats tab shows it: folders first, then the chats +/// filed right here. Search and the archive look through everything below. +struct ChatFolderList: View { + @Environment(AppModel.self) private var app + @Environment(Session.self) private var session + let folderId: String? + @State private var query = "" + @State private var showArchived = false + @State private var naming: FolderNaming? + @State private var name = "" + @State private var deleting: ChatFolder? + @State private var moving: [String]? + + private enum FolderNaming: Identifiable { + case create(parentId: String?) + case rename(ChatFolder) + var id: String { + switch self { + case .create(let p): "create:\(p ?? "")" + case .rename(let f): "rename:\(f.id)" + } + } + } + + private var chats: ChatListStore { session.chats } + private var folder: ChatFolder? { chats.folder(folderId) } + /// Searching or browsing the archive lists chats flat, folders out of the way. + private var flat: Bool { !query.isEmpty || showArchived } + + var body: some View { + List { + if folderId == nil, !session.approvals.queue.isEmpty { + Section("Waiting for you") { + ForEach(session.approvals.queue) { a in + ApprovalCard(approval: a, showThreadLink: true) + .listRowInsets(EdgeInsets(top: 6, leading: 12, bottom: 6, trailing: 12)) } } } - .listStyle(.plain) - .overlay { - if !session.chats.loaded { ProgressView() } - else if rows.isEmpty { - if query.isEmpty { ContentUnavailableView("No chats", systemImage: "bubble.left.and.bubble.right") } - else { ContentUnavailableView.search(text: query) } + if !flat, !subfolders.isEmpty { + Section { + ForEach(subfolders) { f in folderRow(f) } } } - .searchable(text: $query, prompt: "Search chats") - .refreshable { await session.chats.reload() } - .navigationTitle(showArchived ? "Archive" : "Chats") - .toolbar { - ToolbarItem(placement: .topBarLeading) { ConnectionDot() } - ToolbarItem(placement: .topBarTrailing) { - Menu { - Toggle("Show archive", isOn: $showArchived) - Button { Task { await session.chats.setRead(session.chats.chats.map(\.threadId), true) } } label: { - Label("Mark all read", systemImage: "envelope.open") - } - } label: { Image(systemName: "line.3.horizontal.decrease.circle") } - } - ToolbarItem(placement: .topBarTrailing) { - NavigationLink(value: Route.newChat) { Image(systemName: "square.and.pencil") } - .accessibilityLabel("New chat") - } + Section { + ForEach(rows) { chat in chatRow(chat) } } - .navigationDestination(for: Route.self) { route in - switch route { - case .thread(let id): ThreadView(session: session, threadId: id).id(id) - case .newChat: ThreadView(session: session, threadId: nil) - case .mail(let id): MailConversationView(conversationId: id) + } + .listStyle(.plain) + .overlay { + if !chats.loaded { ProgressView() } + else if folderId != nil && folder == nil { + ContentUnavailableView("Folder deleted", systemImage: "folder.badge.minus") + } else if rows.isEmpty && (flat || subfolders.isEmpty) { + if !query.isEmpty { ContentUnavailableView.search(text: query) } + else if folderId != nil { ContentUnavailableView("No chats in this folder", systemImage: "folder") } + else { ContentUnavailableView("No chats", systemImage: "bubble.left.and.bubble.right") } + } + } + .searchable(text: $query, prompt: folder.map { "Search \($0.name)" } ?? "Search chats") + .refreshable { await chats.reload() } + .navigationTitle(showArchived ? "Archive" : folder?.name ?? (folderId == nil ? "Chats" : "")) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + OfflineToolbarItem() + ToolbarItem(placement: .topBarTrailing) { menu } + ToolbarItem(placement: .topBarTrailing) { + NavigationLink(value: Route.newChat(folderId: folderId)) { Image(systemName: "square.and.pencil") } + .accessibilityLabel(folder.map { "New chat in \($0.name)" } ?? "New chat") + } + } + .alert(namingTitle, isPresented: Binding(get: { naming != nil }, set: { if !$0 { naming = nil } })) { + TextField("Name", text: $name) + Button(namingAction) { commitName() } + Button("Cancel", role: .cancel) { naming = nil } + } + .confirmationDialog(deleting.map { "Delete “\($0.name)”?" } ?? "", + isPresented: Binding(get: { deleting != nil }, set: { if !$0 { deleting = nil } }), + titleVisibility: .visible) { + Button("Delete folder", role: .destructive) { + if let f = deleting { delete(f) } + } + } message: { + Text("Its chats and folders move up a level. No chat is deleted.") + } + .sheet(isPresented: Binding(get: { moving != nil }, set: { if !$0 { moving = nil } })) { + if let ids = moving { MoveToFolderSheet(threadIds: ids).environment(session) } + } + } + + private var menu: some View { + Menu { + if folderId == nil { Toggle("Show archive", isOn: $showArchived) } + Button { startNaming(.create(parentId: folderId)) } label: { + Label(folderId == nil ? "New folder" : "New folder inside", systemImage: "folder.badge.plus") + } + Button { Task { await chats.setRead(scope.map(\.threadId), true) } } label: { + Label("Mark all read", systemImage: "envelope.open") + } + if let f = folder { + Divider() + Button { startNaming(.rename(f)) } label: { Label("Rename folder", systemImage: "pencil") } + Button(role: .destructive) { deleting = f } label: { Label("Delete folder", systemImage: "trash") } + } + } label: { Image(systemName: "line.3.horizontal.decrease.circle") } + } + + private func folderRow(_ f: ChatFolder) -> some View { + NavigationLink(value: Route.folder(f.id)) { + HStack(spacing: 10) { + Image(systemName: "folder").foregroundStyle(Color.accentColor).frame(width: 22) + Text(f.name).lineLimit(1) + Spacer() + let unread = chats.unreadCount(in: f.id) + if unread > 0 { + Text("\(unread)").font(.caption.weight(.semibold)).foregroundStyle(.secondary) + .accessibilityLabel("\(unread) unread") } } - .task { if !session.chats.loaded { await session.chats.reload() } } + } + .swipeActions(edge: .trailing) { + Button { deleting = f } label: { Label("Delete", systemImage: "trash") }.tint(.red) + Button { startNaming(.rename(f)) } label: { Label("Rename", systemImage: "pencil") } + } + .contextMenu { + Button { startNaming(.rename(f)) } label: { Label("Rename", systemImage: "pencil") } + Button { startNaming(.create(parentId: f.id)) } label: { Label("New folder inside", systemImage: "folder.badge.plus") } + Button(role: .destructive) { deleting = f } label: { Label("Delete", systemImage: "trash") } } } + private func chatRow(_ chat: ChatSummary) -> some View { + let unread = chats.isUnread(chat) + let archived = chats.isArchived(chat) + return NavigationLink(value: Route.thread(chat.threadId)) { + ChatRow(chat: chat, folderName: flat && chat.folderId != folderId ? chats.folder(chat.folderId)?.name : nil) + } + .swipeActions(edge: .leading) { + Button { Task { await chats.setRead([chat.threadId], unread) } } label: { + Label(unread ? "Read" : "Unread", systemImage: unread ? "envelope.open" : "envelope.badge") + } + .tint(.blue) + } + .swipeActions(edge: .trailing) { + Button { Task { await chats.setArchived([chat.threadId], !archived) } } label: { + Label(archived ? "Unarchive" : "Archive", systemImage: "archivebox") + } + .tint(.gray) + Button { moving = [chat.threadId] } label: { Label("Move", systemImage: "folder") } + .tint(.indigo) + Button(role: .destructive) { Task { await chats.delete(chat.threadId) } } label: { + Label("Delete", systemImage: "trash") + } + } + .contextMenu { + Button { moving = [chat.threadId] } label: { Label("Move to folder…", systemImage: "folder") } + Button { Task { await chats.setRead([chat.threadId], unread) } } label: { + Label(unread ? "Mark read" : "Mark unread", systemImage: unread ? "envelope.open" : "envelope.badge") + } + Button { Task { await chats.setArchived([chat.threadId], !archived) } } label: { + Label(archived ? "Unarchive" : "Archive", systemImage: "archivebox") + } + Button(role: .destructive) { Task { await chats.delete(chat.threadId) } } label: { + Label("Delete", systemImage: "trash") + } + } + } + + private var subfolders: [ChatFolder] { chats.childFolders(of: folderId) } + + /// Every chat this level reaches: all of them at the top, the subtree in a folder. + private var scope: [ChatSummary] { + guard let folderId else { return chats.chats } + let ids = chats.subtree(folderId) + return chats.chats.filter { $0.folderId.map(ids.contains) == true } + } + private var rows: [ChatSummary] { let q = query.lowercased() - return session.chats.chats.filter { c in - (showArchived == session.chats.isArchived(c)) + let base = flat ? scope : chats.chats.filter { $0.folderId == folderId } + return base.filter { c in + (showArchived == chats.isArchived(c)) && (q.isEmpty || c.displayTitle.lowercased().contains(q) || (c.preview ?? "").lowercased().contains(q)) } } + + private var namingTitle: String { + switch naming { + case .rename: "Rename folder" + case .create(let p): p == nil ? "New folder" : "New folder in \(chats.folder(p)?.name ?? "folder")" + case nil: "" + } + } + + private var namingAction: String { + if case .rename = naming { return "Rename" } + return "Create" + } + + private func startNaming(_ n: FolderNaming) { + if case .rename(let f) = n { name = f.name } else { name = "" } + naming = n + } + + private func commitName() { + let n = name.trimmingCharacters(in: .whitespacesAndNewlines) + let what = naming + naming = nil + guard !n.isEmpty, let what else { return } + Task { + switch what { + case .create(let p): await chats.createFolder(n, in: p) + case .rename(let f): if n != f.name { await chats.renameFolder(f.id, n) } + } + } + } + + private func delete(_ f: ChatFolder) { + deleting = nil + // Deleting the folder on screen: step back out of it first. + if f.id == folderId, app.chatPath.last == .folder(f.id) { app.chatPath.removeLast() } + Task { await chats.deleteFolder(f.id) } + } +} + +/// Pick where chats go: the top level or any folder, with a way to make one. +struct MoveToFolderSheet: View { + @Environment(Session.self) private var session + @Environment(\.dismiss) private var dismiss + let threadIds: [String] + @State private var creating = false + @State private var name = "" + + private var current: String?? { + let set = Set(session.chats.chats.filter { threadIds.contains($0.threadId) }.map(\.folderId)) + return set.count == 1 ? set.first : nil + } + + var body: some View { + NavigationStack { + List { + destination(nil, label: "Top level", icon: "tray", depth: 0) + ForEach(session.chats.folderTree, id: \.folder.id) { row in + destination(row.folder.id, label: row.folder.name, icon: "folder", depth: row.depth) + } + Button { name = ""; creating = true } label: { Label("New folder…", systemImage: "folder.badge.plus") } + } + .navigationTitle("Move to") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .cancellationAction) { Button("Cancel") { dismiss() } } + } + .alert("New folder", isPresented: $creating) { + TextField("Name", text: $name) + Button("Create and move") { + let n = name.trimmingCharacters(in: .whitespacesAndNewlines) + guard !n.isEmpty else { return } + Task { + if let id = await session.chats.createFolder(n, in: nil) { move(to: id) } + } + } + Button("Cancel", role: .cancel) {} + } + } + .presentationDetents([.medium, .large]) + } + + private func destination(_ id: String?, label: String, icon: String, depth: Int) -> some View { + Button { move(to: id) } label: { + HStack(spacing: 10) { + Image(systemName: icon).foregroundStyle(Color.accentColor).frame(width: 22) + Text(label).foregroundStyle(.primary).lineLimit(1) + Spacer() + if current == .some(id) { Image(systemName: "checkmark").foregroundStyle(Color.accentColor) } + } + .padding(.leading, CGFloat(depth) * 18) + } + .tint(.primary) + } + + private func move(to id: String?) { + dismiss() + Task { await session.chats.moveChats(threadIds, to: id) } + } } struct ChatRow: View { @Environment(Session.self) private var session let chat: ChatSummary + /// Where the chat is filed, shown when the list mixes folders (search, archive). + var folderName: String? = nil var body: some View { let unread = session.chats.isUnread(chat) @@ -113,6 +342,9 @@ struct ChatRow: View { if let p = chat.preview, !p.isEmpty { Text(p).font(.subheadline).foregroundStyle(.secondary).lineLimit(2) } + if let folderName { + Label(folderName, systemImage: "folder").font(.caption).foregroundStyle(.secondary) + } } } .padding(.vertical, 2) @@ -130,6 +362,20 @@ struct ConnectionDot: View { } } +/// Nothing while connected or reconnecting; a red badge once the server is unreachable. +struct OfflineToolbarItem: ToolbarContent { + @Environment(Session.self) private var session + var body: some ToolbarContent { + if session.connection.status == .offline { + ToolbarItem(placement: .topBarLeading) { + Image(systemName: "wifi.slash") + .foregroundStyle(.red) + .accessibilityLabel("Offline") + } + } + } +} + enum RelativeTime { static func short(_ ms: Double) -> String { let date = Date(timeIntervalSince1970: ms / 1000) diff --git a/ios/Stem/Views/MailViews.swift b/ios/Stem/Views/MailViews.swift index c86faf1e..50a9a522 100644 --- a/ios/Stem/Views/MailViews.swift +++ b/ios/Stem/Views/MailViews.swift @@ -58,8 +58,9 @@ struct MailListView: View { } .refreshable { await session.mail.reload() } .navigationTitle("Mail") + .navigationBarTitleDisplayMode(.inline) .toolbar { - ToolbarItem(placement: .topBarLeading) { ConnectionDot() } + OfflineToolbarItem() ToolbarItem(placement: .topBarTrailing) { Button { composing = true } label: { Image(systemName: "square.and.pencil") } .accessibilityLabel("New mail") @@ -69,7 +70,8 @@ struct MailListView: View { switch route { case .mail(let id): MailConversationView(conversationId: id) case .thread(let id): ThreadView(session: session, threadId: id).id(id) - case .newChat: ThreadView(session: session, threadId: nil) + case .newChat(let folderId): ThreadView(session: session, threadId: nil, folderId: folderId) + case .folder(let id): ChatFolderView(folderId: id) } } .sheet(isPresented: $composing) { @@ -135,11 +137,19 @@ struct MailConversationView: View { /// over the Reply button; asking about a question starts a reply. @State private var formHost = MdxFormHost(place: .below) @State private var formError: String? + /// The reply whose Share sheet is open. + @State private var sharing: ShareRequest? private var conversation: MailConversation? { session.mail.conversation(conversationId) } private var working: Bool { conversation?.status == "working" } private var lead: String { conversation?.participants.first ?? "normal" } + /// A persona's reply in the user's own thread. Mail with another person + /// isn't ours to publish, and the memory check is the user's memory. + private func shareable(_ item: MailItem) -> Bool { + item.from != "user" && item.approval == nil && conversation?.external == nil && conversation?.owner == nil + } + var body: some View { let items = session.mail.items(conversationId) let layout = MailThread.layout(items) @@ -153,7 +163,10 @@ struct MailConversationView: View { ForEach(layout.entries) { entry in Group { if layout.open.contains(entry.id) != flipped.contains(entry.id) { - MailItemCard(item: entry.item, onFold: entry.item.approval?.status == "pending" ? nil : { flip(entry.id) }) + MailItemCard(item: entry.item, onFold: entry.item.approval?.status == "pending" ? nil : { flip(entry.id) }, + onShare: shareable(entry.item) ? { + sharing = ShareRequest(source: .mail(conversationId: conversationId), anchorId: entry.item.id) + } : nil) .environment(\.mdxActions, entry.item.from == "user" ? nil : mdxActions) .environment(\.mdxIsLatest, entry.id == newestFromThem && !working) .environment(\.mdxSource, MdxSource(key: "mail:\(entry.id)", scrollId: entry.id, order: entry.item.at)) @@ -201,6 +214,7 @@ struct MailConversationView: View { .background(.bar) } } + .sheet(item: $sharing) { r in ShareLinkSheet(request: r).environment(session) } .sheet(item: $replying) { r in MailReplySheet(conversationId: conversationId, focusRequest: r.focus ? 1 : 0).environment(session) } @@ -232,6 +246,8 @@ struct MailConversationView: View { if !session.mail.loaded { await session.mail.reload() } await markRead() } + .task { await session.mail.open(conversationId) } + .onDisappear { session.mail.close(conversationId) } // A reply that lands while the conversation is open has been read. .onChange(of: conversation?.userUpdatedAt) { _, _ in Task { await markRead() } } } @@ -442,6 +458,8 @@ struct MailItemCard: View { let item: MailItem /// Fold this mail back to one line (absent where it can't fold). var onFold: (() -> Void)? = nil + /// Opens the Share sheet for this reply (absent where it can't be shared). + var onShare: (() -> Void)? = nil var body: some View { let mine = item.from == "user" @@ -464,6 +482,12 @@ struct MailItemCard: View { if item.stale == true { Text("Late reply — written before your newest message").font(.caption).foregroundStyle(.orange) } + if item.statusReply == true { + Text("Live status — answered on the spot while the work carries on").font(.caption).foregroundStyle(.secondary) + } + if item.statusQuery == true { + Text("Answered on the spot, not passed to the personas at work").font(.caption).foregroundStyle(.secondary) + } if item.filed == true { Text("Arrived after the answer — filed without starting another turn").font(.caption).foregroundStyle(.secondary) } @@ -476,6 +500,10 @@ struct MailItemCard: View { .frame(maxWidth: .infinity, alignment: .leading) .background(mine ? Color.accentColor.opacity(0.10) : Color(.secondarySystemBackground).opacity(0.7), in: RoundedRectangle(cornerRadius: 14)) + .contextMenu { + Button("Copy") { UIPasteboard.general.string = item.body } + if let onShare { Button("Share as Link…", systemImage: "square.and.arrow.up") { onShare() } } + } } } diff --git a/ios/Stem/Views/ShareSheet.swift b/ios/Stem/Views/ShareSheet.swift new file mode 100644 index 00000000..7a84b716 --- /dev/null +++ b/ios/Stem/Views/ShareSheet.swift @@ -0,0 +1,190 @@ +import SwiftUI + +/// Publish an answer, the question and answer, or the whole chat / mail thread +/// as a public link (docs/public-share-design.md). Opened from an answer's +/// context menu. The preview is the snapshot as it will be uploaded. +struct ShareLinkSheet: View { + @Environment(Session.self) private var session + @Environment(\.dismiss) private var dismiss + let request: ShareRequest + + @State private var status: ShareStatus? + @State private var scope: ShareScope = .pair + @State private var includeModel = true + @State private var includeAttachments = false + @State private var ttlDays = ShareLimits.ttlChoices[0] + @State private var preview: ShareDraftPreview? + @State private var cardOn = true + @State private var cardTitle: String? + @State private var cardDesc: String? + @State private var error: String? + @State private var busy = false + @State private var link: SharedLink? + @State private var revoked = false + @State private var copied = false + + private var title: String { cardTitle ?? preview?.card?.title ?? "" } + private var desc: String { cardDesc ?? preview?.card?.description ?? "" } + + var body: some View { + NavigationStack { + Form { + if let link { done(link) } else { compose } + } + .navigationTitle(link == nil ? "Share as a link" : (revoked ? "Link revoked" : "Link ready")) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .cancellationAction) { + Button(link == nil ? "Cancel" : "Done") { dismiss() } + } + if link == nil { + ToolbarItem(placement: .confirmationAction) { + Button(busy ? "Publishing…" : "Create link") { Task { await publish() } } + .disabled(busy || preview == nil || status?.available != true) + } + } + } + } + .task { await loadStatus() } + .task(id: "\(scope.rawValue)|\(includeModel)|\(includeAttachments)") { await loadPreview() } + } + + @ViewBuilder private var compose: some View { + if let status, !status.available { + Section { + Label(status.message ?? "Sharing is not available.", systemImage: "exclamationmark.circle") + if status.code == "no_address" { + Text("Verify an address in the desktop app: Settings → Features → Your email address.") + .font(.footnote).foregroundStyle(.secondary) + } + } + } + Section { + Picker("What to share", selection: $scope) { + Text(request.source.isMail ? "Whole thread" : "Whole chat").tag(ShareScope.thread) + Text("Q & A").tag(ShareScope.pair) + Text("Answer").tag(ShareScope.answer) + } + .pickerStyle(.segmented) + } + Section("Preview") { + if let preview { + Text(preview.snapshot.title).font(.headline) + ForEach(Array(preview.snapshot.messages.enumerated()), id: \.offset) { _, m in + VStack(alignment: .leading, spacing: 4) { + HStack { + Text(m.role == "user" ? "Question" : (m.author ?? "Stem")).font(.caption.weight(.semibold)) + if let model = m.model { Text(model).font(.caption).foregroundStyle(.secondary) } + } + if m.role == "user" { Text(m.text) } else { MarkdownView(m.text) } + if let n = m.images?.count, n > 0 { + Text(n == 1 ? "1 picture" : "\(n) pictures").font(.caption).foregroundStyle(.secondary) + } + } + } + } else if error == nil { + ProgressView() + } + } + Section { + Toggle("Show which model answered", isOn: $includeModel) + if includeAttachments || (preview?.omittedAttachments ?? 0) > 0 { + Toggle("Include my pictures", isOn: $includeAttachments) + } + Picker("Expires", selection: $ttlDays) { + ForEach(ShareLimits.ttlChoices, id: \.self) { d in Text(d == 1 ? "In 1 day" : "In \(d) days").tag(d) } + } + } footer: { + Text("Tool steps, memory cards and in-app links are left out. Anyone with the link can read the preview above.") + } + Section { + Toggle("Preview in chat apps", isOn: $cardOn) + if cardOn { + TextField("Title", text: Binding(get: { title }, set: { cardTitle = String($0.prefix(ShareLimits.cardTitleMax)) })) + TextField("Description", text: Binding(get: { desc }, set: { cardDesc = String($0.prefix(ShareLimits.cardDescMax)) }), + axis: .vertical) + .lineLimit(2...4) + } + } footer: { + if cardOn { + Text("Shown when the link is pasted in a chat app. Unlike the page, this preview is not encrypted. Search engines are told not to index any of it.") + } + } + if let error { Section { Text(error).foregroundStyle(.red) } } + } + + @ViewBuilder private func done(_ link: SharedLink) -> some View { + if revoked { + Section { Text("The link no longer opens.") } + } else { + Section { + Text(link.url).font(.footnote.monospaced()).textSelection(.enabled) + if let url = URL(string: link.url) { + ShareLink(item: url) { Label("Share…", systemImage: "square.and.arrow.up") } + } + Button { + UIPasteboard.general.string = link.url + copied = true + } label: { Label(copied ? "Copied" : "Copy link", systemImage: copied ? "checkmark" : "doc.on.doc") } + } footer: { + Text("Works until \(Date(timeIntervalSince1970: link.expiresAt / 1000).formatted(date: .long, time: .omitted)). The key is the part after #, so only people you give the link to can read it.") + } + Section { + Button("Revoke", role: .destructive) { Task { await revoke(link) } }.disabled(busy) + } + } + if let error { Section { Text(error).foregroundStyle(.red) } } + } + + private var options: JSONValue { + var o: [String: JSONValue] = [ + "scope": .string(scope.rawValue), + "anchorId": .string(request.anchorId), + "includeModel": .bool(includeModel), + "includeAttachments": .bool(includeAttachments), + "ttlDays": .number(Double(ttlDays)) + ] + o["card"] = cardOn ? .object(["title": .string(title), "description": .string(desc)]) : .null + return .object(o) + } + + private func loadStatus() async { + do { status = try await session.client.call("share:status", as: ShareStatus.self) } + catch { self.error = error.localizedDescription } + } + + private func loadPreview() async { + preview = nil + do { + preview = try await session.client.call("share:preview", [request.source.json, options], as: ShareDraftPreview.self) + error = nil + } catch { + self.error = error.localizedDescription + } + } + + private func publish() async { + busy = true + defer { busy = false } + do { + let made = try await session.client.call("share:create", [request.source.json, options], as: SharedLink.self) + link = made + error = nil + UIPasteboard.general.string = made.url + copied = true + } catch { + self.error = error.localizedDescription + } + } + + private func revoke(_ link: SharedLink) async { + busy = true + defer { busy = false } + do { + try await session.client.run("share:revoke", [.string(link.id)]) + revoked = true + } catch { + self.error = error.localizedDescription + } + } +} diff --git a/ios/Stem/Views/ThreadView.swift b/ios/Stem/Views/ThreadView.swift index 5d5b54ad..9d10ceac 100644 --- a/ios/Stem/Views/ThreadView.swift +++ b/ios/Stem/Views/ThreadView.swift @@ -10,9 +10,14 @@ struct ThreadView: View { /// Forms in this chat send from the answer bar over the composer. @State private var formHost = MdxFormHost(place: .below) @State private var focusRequest = 0 + /// Whether the list sits at its end, so growth below follows it there. + /// Not observed: re-rendering on every scroll step looped the layout. + @State private var follow = ScrollFollow() + /// The answer whose Share sheet is open. + @State private var sharing: ShareRequest? - init(session: Session, threadId: String?) { - _store = State(initialValue: ThreadStore(session: session, threadId: threadId)) + init(session: Session, threadId: String?, folderId: String? = nil) { + _store = State(initialValue: ThreadStore(session: session, threadId: threadId, folderId: folderId)) let key = threadId.map { "chat:\($0)" } ?? "chat:new" _draftKey = State(initialValue: key) _draft = State(initialValue: DraftStore.load(key)) @@ -24,20 +29,36 @@ struct ThreadView: View { LazyVStack(alignment: .leading, spacing: 14) { if store.threadId == nil && store.messages.isEmpty { newChatHeader } if store.loading { ProgressView().frame(maxWidth: .infinity).padding() } + let rows = Array(store.messages.enumerated()) + // The split moves in steps, so a row isn't rebuilt on every message. + let split = max(0, (rows.count - Self.eagerTail) / Self.eagerStep * Self.eagerStep) let latest = latestReplyId - ForEach(Array(store.messages.enumerated()), id: \.element.id) { i, m in - MessageRow(message: m, store: store, isLatest: m.id == latest, submit: submitFromReply) - .environment(\.mdxSource, MdxSource(key: "chat:\(store.threadId ?? ""):\(m.turnId ?? m.id)", - scrollId: m.id, order: Double(i))) - .id(m.id) - } - if store.busy { - LiveActivity(label: store.activityLabel, activities: store.activities) + ForEach(rows[..= g.contentSize.height - 80 + } action: { _, now in follow.atBottom = now } + // The bottom anchor alone stopped short of a new reply in a long + // chat; scrolling to the (always drawn) last row lands exactly. + .onChange(of: tailSignature) { old, new in + guard new.userCount > old.userCount || follow.atBottom, let last = lastRowId else { return } + DispatchQueue.main.async { proxy.scrollTo(last, anchor: .bottom) } + } .environment(\.mdxFormHost, formHost) .onAppear { formHost.scrollTo = { id in withAnimation { proxy.scrollTo(id, anchor: .top) } } @@ -74,6 +95,7 @@ struct ThreadView: View { } } } + .sheet(item: $sharing) { r in ShareLinkSheet(request: r).environment(session) } .navigationTitle(navTitle) .navigationBarTitleDisplayMode(.inline) .toolbar { @@ -108,6 +130,31 @@ struct ThreadView: View { } } + /// At least this many of the last messages are always drawn. + private static let eagerTail = 12 + private static let eagerStep = 8 + private static let activityId = "thread-activity" + + /// The row drawn last. A spacer row as the target instead hung the layout. + private var lastRowId: String? { store.busy ? Self.activityId : store.messages.last?.id } + + /// What changes at the end of the list: a new row, a growing reply, the activity line. + private var tailSignature: TailSignature { + TailSignature(count: store.messages.count, lastId: store.messages.last?.id, + lastLength: store.messages.last?.content.count ?? 0, busy: store.busy, + userCount: store.messages.lazy.filter { $0.role == "user" }.count) + } + + private func row(_ m: ChatMessage, _ i: Int, latest: String?) -> some View { + MessageRow(message: m, store: store, isLatest: m.id == latest, submit: submitFromReply, + onShare: { turnId in + if let id = store.threadId { sharing = ShareRequest(source: .chat(threadId: id), anchorId: turnId) } + }) + .environment(\.mdxSource, MdxSource(key: "chat:\(store.threadId ?? ""):\(m.turnId ?? m.id)", + scrollId: m.id, order: Double(i))) + .id(m.id) + } + private var unreadOnScreen: Bool { guard phase == .active, let id = store.threadId, session.chats.inbox.entries[id]?.forcedUnread != true, @@ -127,6 +174,9 @@ struct ThreadView: View { private var newChatHeader: some View { VStack(alignment: .leading, spacing: 14) { + if let folder = session.chats.folder(store.folderId) { + Label("In \(folder.name)", systemImage: "folder").font(.footnote).foregroundStyle(.secondary) + } if !session.chatPersonas.isEmpty { Text("Talk to").font(.footnote).foregroundStyle(.secondary) ScrollView(.horizontal, showsIndicators: false) { @@ -196,6 +246,18 @@ struct ThreadView: View { } } +private final class ScrollFollow { + var atBottom = true +} + +private struct TailSignature: Equatable { + var count: Int + var lastId: String? + var lastLength: Int + var busy: Bool + var userCount: Int +} + struct ErrorBanner: View { let text: String var dismiss: () -> Void @@ -233,6 +295,8 @@ struct MessageRow: View { let store: ThreadStore var isLatest = false var submit: ((String) -> Void)? + /// Opens the Share sheet for this answer's turn. + var onShare: ((String) -> Void)? var body: some View { switch message.role { @@ -272,7 +336,12 @@ struct MessageRow: View { ForEach(message.images ?? [], id: \.id) { ref in GeneratedImage(ref: ref, store: store) } if let sources = message.sources, !sources.isEmpty { SourcesList(sources: sources) } } - .contextMenu { Button("Copy") { UIPasteboard.general.string = message.content } } + .contextMenu { + Button("Copy") { UIPasteboard.general.string = message.content } + if let onShare, let turn = message.turnId, !message.content.isEmpty, !store.busy || !isLatest { + Button("Share as Link…", systemImage: "square.and.arrow.up") { onShare(turn) } + } + } } } } diff --git a/ios/StemTests/MdxTreeTests.swift b/ios/StemTests/MdxTreeTests.swift index d7052b19..92dc0e93 100644 --- a/ios/StemTests/MdxTreeTests.swift +++ b/ios/StemTests/MdxTreeTests.swift @@ -49,6 +49,14 @@ final class MdxTreeTests: XCTestCase { } final class MdxDataTests: XCTestCase { + func testCompleteRowsClosesAfterLastCompleteRow() { + XCTAssertEqual(MdxDataParse.completeRows(#"[{"a":1},{"a":2},{"a""#), #"[{"a":1},{"a":2}]"#) + XCTAssertEqual(MdxDataParse.completeRows(#"[{"s":"}]\"{"},{"t":[1,{"u":2}]},{"v":"x"#), #"[{"s":"}]\"{"},{"t":[1,{"u":2}]}]"#) + XCTAssertEqual(MdxDataParse.completeRows(#"[{"a":1}]"#), #"[{"a":1}]"#) + XCTAssertNil(MdxDataParse.completeRows(#"[{"a":"#)) + XCTAssertNil(MdxDataParse.completeRows(#"{"columns":["a"],"rows":[[1]"#)) + } + func testChartColumnsKeepOrderAndPickTextX() throws { let d = try XCTUnwrap(MdxChartData.from(#"[{"month":"Jan","power":92,"water":"31"},{"month":"Feb","power":88,"water":29}]"#)) XCTAssertEqual(d.xName, "month") diff --git a/ios/project.yml b/ios/project.yml index d7ab8f7f..b1c7329d 100644 --- a/ios/project.yml +++ b/ios/project.yml @@ -16,9 +16,12 @@ targets: platform: iOS sources: - Stem - # The desktop's own mermaid build, so a draws the same on the + # The desktop's own pele build, so a draws the same on the # phone; needs `npm install` in the repo first, as the desktop does. - - path: ../node_modules/mermaid/dist/mermaid.min.js + - path: ../node_modules/pele/dist/pele.global.js + buildPhase: resources + # The desktop's DOMPurify, which sanitizes pele's SVG before it is shown. + - path: ../node_modules/dompurify/dist/purify.min.js buildPhase: resources # The on-device speech-to-text core, shared with the Mac helper's dictation. - path: ../native/mac/stem-computer/Transcriber.swift diff --git a/native/linux/gnome-extension/stem-computer@awantech.sk/core.js b/native/linux/gnome-extension/stem-computer@awantech.sk/core.js new file mode 100644 index 00000000..3284b3b6 --- /dev/null +++ b/native/linux/gnome-extension/stem-computer@awantech.sk/core.js @@ -0,0 +1,355 @@ +// The working half of Stem's GNOME Shell extension: everything Stem asks +// the compositor for (see extension.js for the interface and the trust +// model). Kept apart from the loader so a newer copy can be imported in +// place — `Reload` — instead of waiting for the next login, which is the +// only time GNOME loads an extension's own file. + +import Clutter from 'gi://Clutter'; +import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; +import Meta from 'gi://Meta'; +import Shell from 'gi://Shell'; +import * as Config from 'resource:///org/gnome/shell/misc/config.js'; +import * as Main from 'resource:///org/gnome/shell/ui/main.js'; + +/** Bumped when the helper needs something this file did not have. */ +export const PROTOCOL = 3; + +Gio._promisify(Shell.Screenshot.prototype, 'screenshot_area'); +Gio._promisify(Shell.Screenshot, 'composite_to_stream'); + +/** A refusal the caller reads as is. */ +export class Failure extends Error {} + +const LISTED_TYPES = new Set([ + Meta.WindowType.NORMAL, + Meta.WindowType.DIALOG, + Meta.WindowType.MODAL_DIALOG, + Meta.WindowType.UTILITY +]); + +function rect(r) { + return { x: r.x, y: r.y, width: r.width, height: r.height }; +} + +export class Core { + /** + * @param {{ dir: string, emitHumanInput(kind: string): void }} host + * the private runtime folder (pictures, debug flag) and the loader's signal + */ + constructor(host) { + this._dir = host.dir; + this._emitHumanInput = host.emitHumanInput; + this._pointer = null; + this._keyboard = null; + this._filter = 0; + this._fired = false; + /** When this extension last sent pointer / key input, to tell its own events from the person's. */ + this._sent = { pointer: 0, key: 0 }; + this._callerPid = -1; + } + + /** Is the kill switch armed (a run is driving the screen)? */ + get watching() { + return this._filter !== 0 && !this._fired; + } + + /** Arm the kill switch (the loader carries it across a reload). */ + watch() { + this._watch(); + } + + /** Stop watching and let go of the virtual devices (disable, or a reload). */ + destroy() { + this._unwatch(); + this._pointer = null; + this._keyboard = null; + } + + /** One request; `callerPid` is the process behind the D-Bus caller. */ + handle(req, callerPid) { + this._callerPid = callerPid; + return this._handle(req); + } + + async _handle(req) { + switch (req.op) { + case 'info': + return this._info(); + case 'windows': + return { windows: this._windows() }; + case 'shot': + return req.window !== undefined ? this._shotWindow(req.window) : this._shotArea(req.area); + case 'motion': + this._sent.pointer = Date.now(); + (await this._ensurePointer()).notify_absolute_motion(GLib.get_monotonic_time(), req.x, req.y); + return {}; + case 'button': + this._sent.pointer = Date.now(); + (await this._ensurePointer()).notify_button( + GLib.get_monotonic_time(), + req.button, + req.press ? Clutter.ButtonState.PRESSED : Clutter.ButtonState.RELEASED + ); + return {}; + case 'scroll': { + const dir = { + up: Clutter.ScrollDirection.UP, + down: Clutter.ScrollDirection.DOWN, + left: Clutter.ScrollDirection.LEFT, + right: Clutter.ScrollDirection.RIGHT + }[req.dir]; + if (dir === undefined) throw new Failure('scroll direction must be up, down, left or right.'); + const p = await this._ensurePointer(); + this._sent.pointer = Date.now(); + for (let i = 0; i < Math.max(1, Math.min(req.steps ?? 1, 50)); i++) + p.notify_discrete_scroll(GLib.get_monotonic_time(), dir, Clutter.ScrollSource.WHEEL); + return {}; + } + case 'keyval': + this._sent.key = Date.now(); + (await this._ensureKeyboard()).notify_keyval( + GLib.get_monotonic_time(), + req.keyval, + req.press ? Clutter.KeyState.PRESSED : Clutter.KeyState.RELEASED + ); + return {}; + case 'commit': + // Text for the focused field through the input method, the way the + // on-screen keyboard types: any character, whatever the layout. Only + // a client that takes input-method text (text-input-v3) has a focus. + if (!Main.inputMethod?.currentFocus) return { committed: false }; + this._sent.key = Date.now(); + Main.inputMethod.commit(req.text); + return { committed: true }; + case 'place': { + // Wayland lets no app place its own window: the run's banner asks + // here to sit at the top of the screen, above everything, on every + // workspace. Only a window of the app that asked moves: the helper + // that calls is a child of Stem, whose window it places. + const w = this._find(req.window); + const pid = w.get_pid(); + if (pid !== this._callerPid && pid !== this._parentOf(this._callerPid)) throw new Failure('Only Stem\'s own windows can be placed.'); + if (!w.is_above()) w.make_above(); + if (!w.is_on_all_workspaces()) w.stick(); + w.move_frame(false, Math.round(req.x), Math.round(req.y)); + return {}; + } + case 'pointer': { + const [x, y] = global.get_pointer(); + return { x, y }; + } + case 'watch': + if (req.on) this._watch(); + else this._unwatch(); + return {}; + default: + throw new Failure(`unknown op "${req.op}"`); + } + } + + /** A process's parent, from /proc, or -1. */ + _parentOf(pid) { + try { + const [, bytes] = GLib.file_get_contents(`/proc/${pid}/stat`); + const stat = new TextDecoder().decode(bytes); + // "pid (comm) state ppid …" — comm may hold spaces and parentheses. + return Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[1]); + } catch { + return -1; + } + } + + _info() { + const display = global.display; + const monitors = []; + for (let i = 0; i < display.get_n_monitors(); i++) { + monitors.push({ ...rect(display.get_monitor_geometry(i)), scale: display.get_monitor_scale(i) }); + } + const [x, y] = global.get_pointer(); + return { + protocol: PROTOCOL, + shell: Config.PACKAGE_VERSION, + primary: display.get_primary_monitor(), + monitors, + pointer: { x, y } + }; + } + + /** Top-level windows a person would call a window, the top of the stack first. */ + _windows() { + const display = global.display; + const active = global.workspace_manager.get_active_workspace(); + const tracker = Shell.WindowTracker.get_default(); + const focus = display.focus_window; + const all = global + .get_window_actors() + .map((a) => a.get_meta_window()) + .filter((w) => w && LISTED_TYPES.has(w.get_window_type()) && !w.is_override_redirect()); + return display + .sort_windows_by_stacking(all) + .reverse() + .map((w) => { + // An app GNOME cannot place gets a made-up "window:N" one; its class names it better. + let app = tracker.get_window_app(w); + if (app?.get_id()?.startsWith('window:')) app = null; + return { + id: w.get_id(), + title: w.get_title() ?? '', + app: app?.get_name() ?? w.get_wm_class() ?? '', + appId: app?.get_id() ?? w.get_sandboxed_app_id() ?? w.get_gtk_application_id() ?? null, + wmClass: w.get_wm_class() ?? null, + pid: w.get_pid(), + frame: rect(w.get_frame_rect()), + buffer: rect(w.get_buffer_rect()), + minimized: w.minimized, + focused: w === focus, + otherWorkspace: !(w.is_on_all_workspaces() || w.located_on_workspace(active)), + monitor: w.get_monitor(), + scale: display.get_monitor_scale(Math.max(0, w.get_monitor())) + }; + }); + } + + _find(id) { + const w = global + .get_window_actors() + .map((a) => a.get_meta_window()) + .find((x) => x && x.get_id() === id); + if (!w) throw new Failure(`No window ${id} is open now.`); + return w; + } + + _shotFile() { + const path = GLib.build_filenamev([this._dir, `shot-${GLib.uuid_string_random()}.png`]); + const file = Gio.File.new_for_path(path); + const stream = file.replace(null, false, Gio.FileCreateFlags.PRIVATE, null); + return { path, stream }; + } + + /** The screen, or a part of it, in stage coordinates. The picture is in physical pixels. */ + async _shotArea(area) { + const display = global.display; + const a = area ?? rect(display.get_monitor_geometry(display.get_primary_monitor())); + const { path, stream } = this._shotFile(); + try { + await new Shell.Screenshot().screenshot_area(a.x, a.y, a.width, a.height, stream); + } finally { + stream.close(null); + } + return { path, area: a }; + } + + /** + * One window wherever it is in the stack, cropped to its visible frame (no + * client-side shadow). A minimized window paints nothing. + */ + async _shotWindow(id) { + const w = this._find(id); + if (w.minimized) throw new Failure('minimized'); + const actor = w.get_compositor_private(); + if (!actor) throw new Failure('The window has nothing to show yet.'); + const content = actor.paint_to_content(null); + const texture = content?.get_texture(); + if (!texture) throw new Failure('The compositor would not paint that window.'); + const frame = w.get_frame_rect(); + const buffer = w.get_buffer_rect(); + const ratio = texture.get_width() / Math.max(1, buffer.width); + const x = Math.max(0, Math.round((frame.x - buffer.x) * ratio)); + const y = Math.max(0, Math.round((frame.y - buffer.y) * ratio)); + const width = Math.min(texture.get_width() - x, Math.round(frame.width * ratio)); + const height = Math.min(texture.get_height() - y, Math.round(frame.height * ratio)); + const { path, stream } = this._shotFile(); + try { + await Shell.Screenshot.composite_to_stream(texture, x, y, width, height, ratio, null, 0, 0, 1, stream); + } finally { + stream.close(null); + } + return { path, area: rect(frame) }; + } + + _seat() { + return Clutter.get_default_backend().get_default_seat(); + } + + /** + * A new virtual device is added to the seat asynchronously, and the event + * sent before that lands is dropped (a lost first key, with a stray level + * shift on the next): made on first use, then given a moment to settle. + */ + async _settled(device) { + await new Promise((resolve) => + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 150, () => { + resolve(); + return GLib.SOURCE_REMOVE; + }) + ); + return device; + } + + _ensurePointer() { + if (this._pointer) return this._pointer; + this._pointer = this._settled(this._seat().create_virtual_device(Clutter.InputDeviceType.POINTER_DEVICE)); + return this._pointer; + } + + _ensureKeyboard() { + if (this._keyboard) return this._keyboard; + this._keyboard = this._settled(this._seat().create_virtual_device(Clutter.InputDeviceType.KEYBOARD_DEVICE)); + return this._keyboard; + } + + /** + * The kill switch: the person's first press, click, scroll or real pointer + * move while armed. Our own virtual devices are told apart by their device, + * which is not the person's hardware. Reported once per arming. + */ + _watch() { + this._fired = false; + if (this._filter) return; + this._filter = Clutter.Event.add_filter(null, (event) => { + if (this._fired) return Clutter.EVENT_PROPAGATE; + const kind = { + [Clutter.EventType.KEY_PRESS]: 'key', + [Clutter.EventType.BUTTON_PRESS]: 'mouse', + [Clutter.EventType.SCROLL]: 'mouse', + [Clutter.EventType.MOTION]: 'mouse', + [Clutter.EventType.TOUCH_BEGIN]: 'touch' + }[event.type()]; + if (!kind) return Clutter.EVENT_PROPAGATE; + const device = event.get_source_device(); + const ours = this._isOurs(device, kind); + if (this._debug()) { + console.log(`stem-computer: ${kind} from "${device?.get_device_name()}" node=${device?.get_device_node()} type=${device?.get_device_type()} ours=${ours}`); + } + if (ours) return Clutter.EVENT_PROPAGATE; + this._fired = true; + this._emitHumanInput(kind); + return Clutter.EVENT_PROPAGATE; + }); + } + + /** + * Is this event the input this extension just sent? A device with a device + * file is always the person's: a real mouse, or a streaming client's uinput + * device (Sunshine's "libvirtualhid Mouse") — the person is just remote. The + * compositor's own virtual devices (ours, a remote-desktop session's, a + * libei client's) all look alike, so one of those counts as ours only right + * after we sent that kind of input; anything else on them is someone else. + */ + _isOurs(device, kind) { + if (!device || device.get_device_node()) return false; + const last = kind === 'key' ? this._sent.key : this._sent.pointer; + return Date.now() - last < 300; + } + + /** `touch $XDG_RUNTIME_DIR/stem-computer/debug` logs every watched event to the journal. */ + _debug() { + return GLib.file_test(GLib.build_filenamev([this._dir, 'debug']), GLib.FileTest.EXISTS); + } + + _unwatch() { + if (this._filter) Clutter.Event.remove_filter(this._filter); + this._filter = 0; + } +} diff --git a/native/linux/gnome-extension/stem-computer@awantech.sk/extension.js b/native/linux/gnome-extension/stem-computer@awantech.sk/extension.js new file mode 100644 index 00000000..5d8af1e3 --- /dev/null +++ b/native/linux/gnome-extension/stem-computer@awantech.sk/extension.js @@ -0,0 +1,174 @@ +// Stem computer control, the GNOME Shell half. On Wayland an app cannot see +// other windows or move the pointer, so the Linux helper +// (native/linux/stem-computer, a gjs child of the Stem desktop app) asks the +// compositor through this extension instead: the window list, a picture of the +// screen or of one window, and a virtual pointer and keyboard. Everything it +// reads about a window's controls (AT-SPI) the helper does itself, out here, +// so a hung app can never stall the shell. +// +// The interface is one method on the shell's own bus name, `Call(token, +// request)` with JSON in and out. The token is made fresh each time the +// extension is enabled and kept in $XDG_RUNTIME_DIR/stem-computer/token +// (0600): a sandboxed app sees neither the file nor, past its bus filter, the +// shell — only a program running unsandboxed as this user gets in, and that +// one could already rewrite the user's files. Pictures go back as PNG files in +// the same private folder, named by the extension, never by the caller. +// +// This file is only the loader: GNOME reads an extension's own file once per +// login, so the work lives in core.js, which `Reload(token)` imports again +// from this extension's own folder — a Stem update takes effect at once +// instead of at the next login. Keep this file small and stable: a change +// here still needs a login. +// +// Nothing here acts on its own. `watch` reports the person's own input while a +// run drives the screen (the kill switch); it reports, it never blocks input. + +import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; +import { Extension } from 'resource:///org/gnome/shell/extensions/extension.js'; + +/** What this loader can do; the helper looks for `Reload` from 1 on. */ +const LOADER = 1; +const OBJECT_PATH = '/sk/awantech/StemComputer'; +const IFACE = ` + + + + + + + + + + + + + + +`; + +export default class StemComputerExtension extends Extension { + enable() { + this._runtime = GLib.build_filenamev([GLib.get_user_runtime_dir(), 'stem-computer']); + GLib.mkdir_with_parents(this._runtime, 0o700); + this._token = GLib.uuid_string_random() + GLib.uuid_string_random(); + const file = Gio.File.new_for_path(GLib.build_filenamev([this._runtime, 'token'])); + const out = file.replace(null, false, Gio.FileCreateFlags.PRIVATE | Gio.FileCreateFlags.REPLACE_DESTINATION, null); + out.write_all(new TextEncoder().encode(this._token), null); + out.close(null); + this._sweepShots(); + this._core = null; + this._protocol = 0; + this._loads = 0; + this._ready = this._load(); + this._dbus = Gio.DBusExportedObject.wrapJSObject(IFACE, this); + this._dbus.export(Gio.DBus.session, OBJECT_PATH); + } + + disable() { + this._core?.destroy(); + this._core = null; + this._dbus?.unexport(); + this._dbus = null; + this._token = null; + try { + Gio.File.new_for_path(GLib.build_filenamev([this._runtime, 'token'])).delete(null); + } catch { + // Already gone. + } + } + + /** + * core.js from this extension's own folder, imported under a fresh URL so a + * copy Stem replaced on disk is read again (the module cache keys by URL). + */ + async _load() { + const url = `${this.dir.get_child('core.js').get_uri()}?load=${++this._loads}-${Date.now()}`; + const mod = await import(url); + const core = new mod.Core({ + dir: this._runtime, + emitHumanInput: (kind) => this._dbus?.emit_signal('HumanInput', new GLib.Variant('(s)', [kind])) + }); + // A reload mid-run must not leave the screen driven with the kill switch + // off: an armed watch moves to the new core before the old one goes. + if (this._core?.watching) core.watch?.(); + this._core?.destroy(); + this._core = core; + this._protocol = mod.PROTOCOL; + this._Failure = mod.Failure; + } + + Version() { + return this._protocol; + } + + LoaderVersion() { + return LOADER; + } + + ReloadAsync([token], invocation) { + if (!this._token || token !== this._token) { + invocation.return_dbus_error('sk.awantech.StemComputer.Denied', 'Wrong token.'); + return; + } + this._ready = this._load(); + this._ready.then( + () => invocation.return_value(new GLib.Variant('(u)', [this._protocol])), + (e) => { + console.warn(`stem-computer: reload failed: ${e?.message ?? e}`); + invocation.return_dbus_error('sk.awantech.StemComputer.ReloadFailed', String(e?.message ?? e)); + } + ); + } + + CallAsync([token, request], invocation) { + if (!this._token || token !== this._token) { + invocation.return_dbus_error('sk.awantech.StemComputer.Denied', 'Wrong token.'); + return; + } + let req; + try { + req = JSON.parse(request); + } catch { + invocation.return_dbus_error('sk.awantech.StemComputer.Invalid', 'Unreadable request.'); + return; + } + const callerPid = this._pidOf(invocation.get_sender()); + this._ready + .then(() => this._core.handle(req, callerPid)) + .then( + (reply) => invocation.return_value(new GLib.Variant('(s)', [JSON.stringify({ ok: true, ...reply })])), + (e) => { + const failure = this._Failure && e instanceof this._Failure; + const error = failure ? e.message : `${e?.name ?? 'Error'}: ${e?.message ?? e}`; + if (!failure) console.warn(`stem-computer: ${req.op}: ${error}`); + invocation.return_value(new GLib.Variant('(s)', [JSON.stringify({ ok: false, error })])); + } + ); + } + + /** The process behind a bus name, or -1. */ + _pidOf(sender) { + try { + const r = Gio.DBus.session.call_sync('org.freedesktop.DBus', '/org/freedesktop/DBus', 'org.freedesktop.DBus', + 'GetConnectionUnixProcessID', new GLib.Variant('(s)', [sender]), new GLib.VariantType('(u)'), Gio.DBusCallFlags.NONE, 1000, null); + return r.deepUnpack()[0]; + } catch { + return -1; + } + } + + /** Leftover pictures from a helper that died before reading them. */ + _sweepShots() { + try { + const dir = Gio.File.new_for_path(this._runtime); + const files = dir.enumerate_children('standard::name', Gio.FileQueryInfoFlags.NONE, null); + let info; + while ((info = files.next_file(null))) { + if (info.get_name().startsWith('shot-')) dir.get_child(info.get_name()).delete(null); + } + } catch { + // Nothing to sweep. + } + } +} diff --git a/native/linux/gnome-extension/stem-computer@awantech.sk/metadata.json b/native/linux/gnome-extension/stem-computer@awantech.sk/metadata.json new file mode 100644 index 00000000..fc87a304 --- /dev/null +++ b/native/linux/gnome-extension/stem-computer@awantech.sk/metadata.json @@ -0,0 +1,9 @@ +{ + "uuid": "stem-computer@awantech.sk", + "name": "Stem computer control", + "description": "Lets the Stem desktop app see the screen and use the mouse and keyboard when you turn on computer control in Stem. It does nothing unless Stem asks.", + "shell-version": ["46", "47", "48", "49", "50"], + "url": "https://stem.awantech.sk", + "version": 5, + "version-name": "5" +} diff --git a/native/linux/stem-computer/a11y.js b/native/linux/stem-computer/a11y.js new file mode 100644 index 00000000..c21e95e9 --- /dev/null +++ b/native/linux/stem-computer/a11y.js @@ -0,0 +1,502 @@ +// AT-SPI: how a run reaches a window that is not in front, the Linux twin of +// the Mac helper's AX.swift and the Windows helper's Uia.cs. Wayland gives an +// app no way to send input to a window behind others, so window mode asks the +// window's accessibility tree instead: press this button, set that field's +// text. Coverage follows the app's own tree: GTK, Qt, Chromium and Firefox +// answer; canvases and games are blank. +// +// Positions: on Wayland an app does not know where its window sits, so every +// extent is read relative to the window (ATSPI_COORD_TYPE_WINDOW) and placed +// on the stage through the window's frame, which the compositor does know. + +import Atspi from 'gi://Atspi'; +import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; +import { HelperError } from './errors.js'; + +const MAX_NODES = 400; +let initialized = false; +let enabledBefore = null; + +function init() { + if (initialized) return; + Atspi.init(); + // A hung app must not hang the helper: 2s per call, then the call fails. + Atspi.set_timeout(2000, 5000); + initialized = true; +} + +/** + * Chromium and Electron build their trees only once an assistive technology + * is said to be listening; GTK always does. Set for the run, put back after. + */ +export function enableToolkits() { + try { + const bus = Gio.DBus.session; + const get = bus.call_sync('org.a11y.Bus', '/org/a11y/bus', 'org.freedesktop.DBus.Properties', 'Get', + new GLib.Variant('(ss)', ['org.a11y.Status', 'IsEnabled']), null, Gio.DBusCallFlags.NONE, 2000, null); + const on = get.recursiveUnpack()[0]; + if (enabledBefore === null) enabledBefore = on; + if (!on) setEnabled(true); + } catch { + // No accessibility bus: the trees stay as they are. + } +} + +function setEnabled(on) { + Gio.DBus.session.call_sync('org.a11y.Bus', '/org/a11y/bus', 'org.freedesktop.DBus.Properties', 'Set', + new GLib.Variant('(ssv)', ['org.a11y.Status', 'IsEnabled', new GLib.Variant('b', on)]), null, Gio.DBusCallFlags.NONE, 2000, null); +} + +export function restoreToolkits() { + if (enabledBefore === false) { + try { + setEnabled(false); + } catch { + // Gone with the session. + } + } + enabledBefore = null; +} + +function safe(fn, fallback = null) { + try { + const v = fn(); + return v === undefined ? fallback : v; + } catch { + return fallback; + } +} + +function children(el) { + const n = safe(() => el.get_child_count(), 0); + const out = []; + for (let i = 0; i < n && i < 2000; i++) { + const c = safe(() => el.get_child_at_index(i)); + if (c) out.push(c); + } + return out; +} + +const roleName = (el) => safe(() => el.get_role_name(), '') ?? ''; +const nameOf = (el) => safe(() => el.get_name(), '') ?? ''; +const has = (el, state) => safe(() => el.get_state_set().contains(state), false); + +/** + * The text an element holds, up to `max` characters. Called through the Text + * interface by name: Accessible has a get_text() of its own that shadows it. + */ +function textOf(el, max) { + if (!safe(() => el.get_text_iface())) return null; + return safe(() => Atspi.Text.prototype.get_text.call(el, 0, Math.min(max, Atspi.Text.prototype.get_character_count.call(el)))); +} + +function clip(s, limit = 80) { + if (s === null || s === undefined) return null; + s = String(s).replace(/\r?\n/g, '⏎').replace(//g, '').trim(); + if (!s) return null; + return s.length > limit ? `${s.slice(0, limit - 1)}…` : s; +} + +/** Extents relative to the window, or null. */ +function extents(el) { + const c = safe(() => el.get_component_iface()); + if (!c) return null; + const r = safe(() => c.get_extents(Atspi.CoordType.WINDOW)); + return r && r.width > 0 && r.height > 0 ? { x: r.x, y: r.y, width: r.width, height: r.height } : null; +} + +const ROLES = { + 'push button': 'button', 'toggle button': 'button', 'push button menu': 'menubutton', 'check box': 'checkbox', + 'radio button': 'radio', entry: 'textfield', 'password text': 'password', 'spin button': 'stepper', + label: 'text', static: 'text', paragraph: 'text', link: 'link', image: 'image', icon: 'image', + 'menu item': 'menuitem', 'check menu item': 'menuitem', 'radio menu item': 'menuitem', menu: 'menu', + 'menu bar': 'menubar', list: 'list', 'list box': 'list', 'list item': 'row', table: 'table', + 'tree table': 'table', tree: 'outline', 'table row': 'row', 'tree item': 'row', 'table cell': 'cell', + 'page tab list': 'tabs', 'page tab': 'tab', 'tool bar': 'toolbar', slider: 'slider', 'combo box': 'combobox', + panel: 'group', filler: 'group', section: 'group', grouping: 'group', frame: 'window', window: 'window', + dialog: 'window', 'scroll bar': 'scrollbar', heading: 'heading', 'document web': 'document', + 'document frame': 'document', 'document text': 'document', 'progress bar': 'progress', + 'status bar': 'statusbar', separator: 'separator', 'tool tip': 'tooltip', 'column header': 'column', + 'row header': 'header', text: 'textfield' +}; + +const STRUCTURAL = new Set([ + 'panel', 'filler', 'section', 'unknown', 'redundant object', 'scroll pane', 'viewport', 'layered pane', + 'split pane', 'root pane', 'glass pane', 'internal frame', 'list', 'list box', 'table', 'tree table', 'tree', + 'tool bar', 'page tab list', 'document web', 'document frame', 'grouping', 'form', 'landmark', 'separator', + 'scroll bar', 'menu bar', 'block quote', 'article', 'application', 'frame', 'window', 'dialog' +]); + +const FIELDS = new Set(['entry', 'password text', 'text', 'combo box', 'spin button', 'document text', 'terminal']); +const PRESS_ACTIONS = ['click', 'press', 'activate', 'toggle', 'jump', 'open', 'select', 'expand or contract', 'expand', 'collapse']; +const MENU_ACTIONS = ['showmenu', 'show menu', 'popup', 'menu', 'context menu']; + +function actionNames(el) { + const a = safe(() => el.get_action_iface()); + if (!a) return []; + const n = safe(() => a.get_n_actions(), 0); + const out = []; + for (let i = 0; i < n; i++) out.push((safe(() => a.get_action_name(i), '') ?? '').toLowerCase()); + return out; +} + +function doNamed(el, wanted) { + const a = safe(() => el.get_action_iface()); + if (!a) return false; + const names = actionNames(el); + for (const w of wanted) { + const i = names.indexOf(w); + if (i >= 0) return safe(() => a.do_action(i), false) !== false; + } + return false; +} + +/** The window's accessible: by process first, then by title (a sandboxed app's process is a proxy's). */ +export function findWindow(win) { + init(); + const desktop = Atspi.get_desktop(0); + const apps = children(desktop); + const tops = (apps) => apps.flatMap((app) => children(app)); + const byPid = apps.filter((app) => safe(() => app.get_process_id()) === win.pid); + const title = win.title ?? ''; + const pick = (cands) => { + if (cands.length === 0) return null; + const named = cands.filter((c) => nameOf(c) === title); + const pool = named.length > 0 ? named : cands; + // The active one, else one whose size matches the frame. + return ( + pool.find((c) => has(c, Atspi.StateType.ACTIVE)) ?? + pool.find((c) => { + const e = extents(c); + return e && Math.abs(e.width - win.frame.width) <= 2 && Math.abs(e.height - win.frame.height) <= 2; + }) ?? + (named.length > 0 || cands.length === 1 ? pool[0] : null) + ); + }; + const mine = tops(byPid); + return pick(mine) ?? (title ? pick(tops(apps).filter((c) => nameOf(c) === title)) : null); +} + +/** One selected window's tree, its element ids, and the actions on them. */ +export class WindowTree { + constructor(win) { + this.win = win; + this.root = null; + this.nodes = new Map(); + this.snapshotTaken = false; + /** The field last focused by id or click, for toolkits that refuse focus. */ + this.chosen = null; + } + + update(win) { + this.win = win; + } + + ensureRoot() { + if (this.root) { + if (safe(() => this.root.get_role_name()) !== null) return this.root; + this.root = null; + } + enableToolkits(); + const root = findWindow(this.win); + if (!root) { + throw new HelperError(`${this.win.app}'s window exposes no accessibility tree Stem can find; clear the window (select_window with no arguments) and use the screen.`); + } + this.root = root; + return root; + } + + /** Window-relative extents → stage rectangle. */ + toStage(e) { + const rootExt = extents(this.root) ?? { x: 0, y: 0 }; + return { x: this.win.frame.x + (e.x - rootExt.x), y: this.win.frame.y + (e.y - rootExt.y), width: e.width, height: e.height }; + } + + /** The window's controls, one line each with a fresh id, positioned in pixels of the current picture. */ + snapshot(maxDepth, toPixel, unitsPerShotPixel) { + const top = this.ensureRoot(); + const lines = []; + let truncated = false; + const started = Date.now(); + const frame = this.win.frame; + const walk = (el, depth, indent) => { + if (this.nodes.size >= MAX_NODES || Date.now() - started > 8000) { + truncated = true; + return; + } + const role = roleName(el); + const e = extents(el); + const stage = e ? this.toStage(e) : null; + if (stage && (stage.x + stage.width < frame.x - 2 || stage.y + stage.height < frame.y - 2 || stage.x > frame.x + frame.width + 2 || stage.y > frame.y + frame.height + 2)) return; + if (e && !has(el, Atspi.StateType.SHOWING) && role !== 'frame') return; + const label = clip(nameOf(el)) ?? clip(safe(() => el.get_description())); + const password = role === 'password text'; + const editable = has(el, Atspi.StateType.EDITABLE) && !!safe(() => el.get_editable_text_iface()); + const valueIface = safe(() => el.get_value_iface()); + let value = null; + if (!password && (FIELDS.has(role) || editable)) { + value = clip(textOf(el, 200)); + } else if (valueIface) { + value = clip(String(safe(() => valueIface.get_current_value(), ''))); + } + const focused = has(el, Atspi.StateType.FOCUSED); + const enabled = has(el, Atspi.StateType.ENABLED) || has(el, Atspi.StateType.SENSITIVE); + const actions = actionNames(el); + // A row with no action of its own is chosen through its list (see pressElement). + const pressable = + actions.some((a) => PRESS_ACTIONS.includes(a)) || + (['list item', 'table row', 'tree item', 'table cell'].includes(role) && !!safe(() => el.get_parent()?.get_selection_iface())); + const structure = STRUCTURAL.has(role) && !label && !focused; + if (!structure && stage && depth > 0) { + const id = this.nodes.size; + this.nodes.set(id, el); + const caps = []; + if (pressable) caps.push('press'); + if (actions.some((a) => MENU_ACTIONS.includes(a))) caps.push('menu'); + if (editable || (valueIface && !password)) caps.push('setvalue'); + let shown = password ? 'password' : (ROLES[role] ?? role.replace(/\s+/g, '')); + if (role === 'text' && !editable) shown = 'text'; + let line = `${' '.repeat(Math.min(indent, 12))}${id} ${shown}`; + if (label) line += ` "${label}"`; + if (value && value !== label && shown !== 'text') line += ` = "${value}"`; + if (focused) line += ' focused'; + if (!enabled) line += ' disabled'; + if (caps.length) line += ` [${caps.join(',')}]`; + const xy = toPixel({ x: stage.x, y: stage.y }); + line += ` (${xy.x},${xy.y} ${Math.round(stage.width / unitsPerShotPixel)}x${Math.round(stage.height / unitsPerShotPixel)})`; + lines.push(line); + } + if (depth >= maxDepth) return; + for (const c of children(el)) walk(c, depth + 1, structure ? indent : indent + 1); + }; + this.nodes.clear(); + this.snapshotTaken = true; + walk(top, 0, 0); + // A Chromium window builds its tree on the first question and fills it + // over a second or two: look again before believing a bare one. + for (let waited = 0; lines.length < 12 && !truncated && waited < 6; waited++) { + GLib.usleep(500_000); + this.nodes.clear(); + lines.length = 0; + walk(top, 0, 0); + } + let head = 'Controls of this window (id role "label" [what it can do] (x,y wxh in window pixels)). Ids are valid until the next snapshot.'; + if (truncated) head += ` Only the first ${this.nodes.size} controls are listed; lower \`depth\` or scroll to see others.`; + if (lines.length === 0) return `${head}\n(none exposed — this window may be a canvas; act on it in screen mode)`; + return `${head}\n${lines.join('\n')}`; + } + + node(id) { + if (!this.snapshotTaken) throw new HelperError('Take a snapshot first; element ids come from it.'); + const el = this.nodes.get(id); + if (!el) throw new HelperError(`No element ${id} in the last snapshot. Take a new snapshot and use one of its ids.`); + return el; + } + + /** The element's own action, or its nearest ancestor's. */ + pressElement(el) { + let at = el; + for (let i = 0; i < 4 && at; i++) { + if (doNamed(at, PRESS_ACTIONS)) return true; + if (at === this.root) break; + at = safe(() => at.get_parent()); + } + // A list or table row is chosen through its container. + const parent = safe(() => el.get_parent()); + const selection = parent && safe(() => parent.get_selection_iface()); + const index = safe(() => el.get_index_in_parent(), -1); + if (selection && index >= 0 && safe(() => selection.select_child(index), false)) return true; + return false; + } + + press(id) { + const el = this.node(id); + if (this.pressElement(el)) return; + if (FIELDS.has(roleName(el)) && this.focusElement(el)) return; + throw new HelperError(`press ${id}: this control offers no action. Focus it, or clear the window and click it in screen mode.`); + } + + focusElement(el) { + const c = safe(() => el.get_component_iface()); + return !!(c && safe(() => c.grab_focus(), false)); + } + + /** + * GTK 4 takes no focus through AT-SPI, so a field focused by id is also + * remembered: typing goes into it even where the toolkit said no. + */ + focus(id) { + const el = this.node(id); + const took = this.focusElement(el); + const editable = has(el, Atspi.StateType.EDITABLE) && !!safe(() => el.get_editable_text_iface()); + this.chosen = editable ? el : null; + if (!took && !editable) throw new HelperError(`focus ${id}: the control would not take focus; take a new snapshot.`); + } + + menu(id) { + if (!doNamed(this.node(id), MENU_ACTIONS)) { + throw new HelperError(`menu ${id}: this control has no context menu action. Clear the window and right-click it in screen mode.`); + } + } + + setValue(id, text) { + const el = this.node(id); + const editable = safe(() => el.get_editable_text_iface()); + if (editable && has(el, Atspi.StateType.EDITABLE)) { + if (safe(() => editable.set_text_contents(text), false)) return; + } + const value = safe(() => el.get_value_iface()); + const n = Number(text); + if (value && Number.isFinite(n) && safe(() => value.set_current_value(n), false)) return; + throw new HelperError(`set_value ${id}: this control does not take a value directly. Focus it and use type instead.`); + } + + /** The deepest control under a stage point, found through the tree (not the screen), so a covered window answers for itself. */ + elementAt(p) { + let at = this.ensureRoot(); + for (let depth = 0; depth < 40; depth++) { + let hit = null; + let best = Infinity; + for (const c of children(at)) { + const e = extents(c); + if (!e || !has(c, Atspi.StateType.SHOWING)) continue; + const r = this.toStage(e); + if (p.x < r.x || p.y < r.y || p.x >= r.x + r.width || p.y >= r.y + r.height) continue; + const area = r.width * r.height; + if (area < best) { + best = area; + hit = c; + } + } + if (!hit) break; + at = hit; + } + return at === this.root ? null : at; + } + + /** A window-mode click: the control's own action; a field takes focus. */ + click(p, button, count) { + const el = this.elementAt(p); + if (!el) throw new HelperError('Nothing Stem can press is at that spot in the window. Take a snapshot, or clear the window and click in screen mode.'); + const role = roleName(el); + if (button === 'right') { + if (doNamed(el, MENU_ACTIONS)) return; + throw new HelperError('That control has no context menu action; right-clicks in a background window need screen mode on Linux.'); + } + if (FIELDS.has(role) || has(el, Atspi.StateType.EDITABLE)) { + const took = this.focusElement(el); + this.chosen = has(el, Atspi.StateType.EDITABLE) ? el : null; + if (took || this.chosen) return; + } + if (this.pressElement(el)) { + if (count >= 2) this.pressElement(el); + return; + } + if (this.focusElement(el)) return; + throw new HelperError(`The ${ROLES[role] ?? role} at that spot offers no action Stem can use in window mode. Take a snapshot, or clear the window and click in screen mode.`); + } + + /** Scroll by the nearest scroll bar's value (page steps), found from the point upward. */ + scroll(p, dir, amount) { + const vertical = dir === 'up' || dir === 'down'; + let at = this.elementAt(p) ?? this.ensureRoot(); + for (let i = 0; i < 10 && at; i++) { + for (const c of children(at)) { + if (roleName(c) !== 'scroll bar') continue; + const horizontal = has(c, Atspi.StateType.HORIZONTAL); + if (horizontal === vertical) continue; + const v = safe(() => c.get_value_iface()); + if (!v) continue; + const e = extents(c); + const span = vertical ? (e?.height ?? 400) : (e?.width ?? 400); + const min = safe(() => v.get_minimum_value(), 0); + const max = safe(() => v.get_maximum_value(), 0); + const step = Math.max(safe(() => v.get_minimum_increment(), 0) || 0, (span * 0.15 * Math.max(1, Math.min(amount, 50)))); + const now = safe(() => v.get_current_value(), 0); + const next = Math.min(max, Math.max(min, now + (dir === 'up' || dir === 'left' ? -step : step))); + if (safe(() => v.set_current_value(next), false)) return; + } + if (at === this.root) break; + at = safe(() => at.get_parent()); + } + throw new HelperError('No scrollable area Stem can move is at that spot in the window. Clear the window and scroll in screen mode.'); + } + + /** The control in this window with keyboard focus, or null. */ + focused() { + const root = this.ensureRoot(); + let found = null; + let visited = 0; + const walk = (el, depth) => { + if (found || depth > 40 || visited++ > 4000) return; + if (has(el, Atspi.StateType.FOCUSED)) { + found = el; + return; + } + for (const c of children(el)) { + if (has(c, Atspi.StateType.SHOWING) || has(c, Atspi.StateType.FOCUSED)) walk(c, depth + 1); + } + }; + walk(root, 0); + return found; + } + + /** Typing into a background window: inserted at the caret of its focused field. */ + type(text) { + const focused = this.focused(); + const el = focused && has(focused, Atspi.StateType.EDITABLE) ? focused : (this.chosen ?? focused); + const editable = el && safe(() => el.get_editable_text_iface()); + if (!editable || !has(el, Atspi.StateType.EDITABLE)) { + throw new HelperError('Nothing in this window that takes text has keyboard focus. Click a field or `focus` its id first, then type.'); + } + // At the caret when the field has focus; at the end of a remembered one. + const end = safe(() => Atspi.Text.prototype.get_character_count.call(el), 0) ?? 0; + const caret = el === focused ? safe(() => Atspi.Text.prototype.get_caret_offset.call(el), -1) ?? -1 : -1; + const at = caret >= 0 && caret <= end ? caret : end; + const bytes = new TextEncoder().encode(text).length; + if (!safe(() => editable.insert_text(at, text, bytes), false)) throw new HelperError('The field would not take the text.'); + } + + /** Return in a background window: the focused control's own activate action. */ + activateFocused() { + const el = this.focused(); + if (el && doNamed(el, ['activate', 'press', 'click'])) return true; + return false; + } +} + +/** The window's readable text for "Ask about this window": static text, field values (never passwords), names of links and images. */ +export function readableText(win, maxChars) { + enableToolkits(); + const root = findWindow(win); + if (!root) return null; + const parts = []; + let total = 0; + let visited = 0; + const started = Date.now(); + const takeText = (s) => { + if (!s) return; + s = String(s).replace(//g, '').trim(); + if (!s || parts[parts.length - 1] === s) return; + parts.push(s); + total += s.length + 1; + }; + const walk = (el, depth) => { + if (total >= maxChars || depth > 60 || visited > 8000 || Date.now() - started > 6000) return; + visited++; + const role = roleName(el); + if (role === 'password text') return; + if (depth > 0 && !has(el, Atspi.StateType.SHOWING)) return; + const kids = children(el); + const text = kids.length === 0 ? textOf(el, 20000) : null; + if (text) takeText(text); + else if (kids.length === 0 && ['label', 'static', 'link', 'list item', 'table cell', 'heading'].includes(role)) takeText(nameOf(el)); + else if (role === 'image' && nameOf(el).trim()) takeText(`[image: ${nameOf(el).trim()}]`); + for (const c of kids) walk(c, depth + 1); + }; + walk(root, 0); + if (parts.length === 0) return null; + const all = parts.join('\n'); + return all.length > maxChars ? `${all.slice(0, maxChars)}\n…` : all; +} diff --git a/native/linux/stem-computer/capture.js b/native/linux/stem-computer/capture.js new file mode 100644 index 00000000..8e520d88 --- /dev/null +++ b/native/linux/stem-computer/capture.js @@ -0,0 +1,128 @@ +// Pictures of the primary monitor, or of the selected window alone, taken by +// the extension and downscaled here, plus the geometry that turns screenshot +// pixels back into stage coordinates (GNOME's logical pixels, what the virtual +// pointer takes). The same contract as the Mac and Windows helpers. + +import GdkPixbuf from 'gi://GdkPixbuf'; +import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; +import * as shell from './shell.js'; +import { HelperError } from './errors.js'; + +/** Longest side of a frame handed to the model (Anthropic's computer-use guidance). */ +const MAX_SIDE = 1568; + +/** Loads the extension's PNG and deletes it. */ +function take(path) { + try { + return GdkPixbuf.Pixbuf.new_from_file(path); + } finally { + try { + Gio.File.new_for_path(path).delete(null); + } catch { + // Already gone. + } + } +} + +/** Scaled into an opaque frame: the JPEG encoder takes no alpha channel. */ +function jpeg(pixbuf, width, height) { + const opaque = GdkPixbuf.Pixbuf.new(GdkPixbuf.Colorspace.RGB, false, 8, width, height); + opaque.fill(0xffffffff); + pixbuf.composite(opaque, 0, 0, width, height, 0, 0, width / pixbuf.get_width(), height / pixbuf.get_height(), GdkPixbuf.InterpType.BILINEAR, 255); + const [, bytes] = opaque.save_to_bufferv('jpeg', ['quality'], ['80']); + return GLib.base64_encode(bytes); +} + +export class Capture { + constructor() { + /** { id, app, title, pid } of the selected window, or null for the screen. */ + this._target = null; + this.reset(); + } + + reset() { + this.lastWidth = 0; + this.lastHeight = 0; + /** Stage units per screenshot pixel. */ + this.unitsPerShotPixel = 1; + this.origin = { x: 0, y: 0 }; + } + + get target() { + return this._target; + } + + set target(value) { + const changed = (this._target === null) !== (value === null) || (this._target && value && this._target.id !== value.id); + this._target = value; + if (changed) this.reset(); + } + + /** The full-resolution picture and the stage rectangle it covers. */ + async grab() { + if (!this._target) { + const r = await shell.call('shot'); + return { pixbuf: take(r.path), area: r.area }; + } + let r; + try { + r = await shell.call('shot', { window: this._target.id }); + } catch (e) { + if (e instanceof HelperError && e.message === 'minimized') { + throw new HelperError(`${this._target.app}'s window is minimized, and GNOME draws nothing for a minimized window. Ask the user to restore it (it can stay behind other windows), or clear the window and use the screen.`); + } + if (e instanceof HelperError && /^No window/.test(e.message)) { + throw new HelperError('The selected window has closed (an action that closed it went through). Run list_windows and select again.'); + } + throw e; + } + return { pixbuf: take(r.path), area: r.area }; + } + + async screenshot() { + const { pixbuf, area } = await this.grab(); + const longest = Math.max(pixbuf.get_width(), pixbuf.get_height()); + const factor = longest > MAX_SIDE ? MAX_SIDE / longest : 1; + const width = Math.max(1, Math.round(pixbuf.get_width() * factor)); + const height = Math.max(1, Math.round(pixbuf.get_height() * factor)); + this.lastWidth = width; + this.lastHeight = height; + this.unitsPerShotPixel = area.width / width; + this.origin = { x: area.x, y: area.y }; + this.area = area; + // `scale`: screen pixels per screenshot pixel, as the other helpers report it. + return { jpegBase64: jpeg(pixbuf, width, height), width, height, scale: pixbuf.get_width() / width }; + } + + async zoom(x, y, w, h) { + if (this.lastWidth <= 0) throw new HelperError('Take a screenshot before zooming.'); + if (!(w >= 1 && h >= 1)) throw new HelperError('The zoom region needs a positive width and height.'); + const { pixbuf } = await this.grab(); + const toFull = pixbuf.get_width() / this.lastWidth; + const left = Math.max(0, Math.floor(x * toFull)); + const top = Math.max(0, Math.floor(y * toFull)); + const right = Math.min(pixbuf.get_width(), Math.round((x + w) * toFull)); + const bottom = Math.min(pixbuf.get_height(), Math.round((y + h) * toFull)); + if (right - left < 1 || bottom - top < 1) throw new HelperError('The zoom region lies outside the screen.'); + const cropped = pixbuf.new_subpixbuf(left, top, right - left, bottom - top); + const factor = MAX_SIDE / Math.max(cropped.get_width(), cropped.get_height()); + const width = Math.max(1, Math.round(cropped.get_width() * factor)); + const height = Math.max(1, Math.round(cropped.get_height() * factor)); + return { jpegBase64: jpeg(cropped, width, height), width, height, zoomed: true }; + } + + /** Screenshot pixels → stage coordinates. */ + toPoint(x, y) { + if (this.lastWidth <= 0) throw new HelperError('Take a screenshot before pointing at it.'); + const px = Math.min(Math.max(x, 0), this.lastWidth - 1); + const py = Math.min(Math.max(y, 0), this.lastHeight - 1); + return { x: this.origin.x + px * this.unitsPerShotPixel, y: this.origin.y + py * this.unitsPerShotPixel }; + } + + /** Stage coordinates → screenshot pixels. */ + toPixel(p) { + if (this.lastWidth <= 0) return { x: Math.round(p.x), y: Math.round(p.y) }; + return { x: Math.round((p.x - this.origin.x) / this.unitsPerShotPixel), y: Math.round((p.y - this.origin.y) / this.unitsPerShotPixel) }; + } +} diff --git a/native/linux/stem-computer/errors.js b/native/linux/stem-computer/errors.js new file mode 100644 index 00000000..de1633e6 --- /dev/null +++ b/native/linux/stem-computer/errors.js @@ -0,0 +1,2 @@ +/** A failure the model should read as is (no exception name in front). */ +export class HelperError extends Error {} diff --git a/native/linux/stem-computer/input.js b/native/linux/stem-computer/input.js new file mode 100644 index 00000000..59cda005 --- /dev/null +++ b/native/linux/stem-computer/input.js @@ -0,0 +1,142 @@ +// Screen-mode input: the extension's virtual pointer and keyboard, which the +// compositor treats like real devices (the window under the pointer gets the +// click, the focused one the keys). + +import GLib from 'gi://GLib'; +import * as shell from './shell.js'; +import { HelperError } from './errors.js'; +import { KEY, charKeysym, parseCombo } from './keys.js'; + +const BUTTONS = { left: 1, middle: 2, right: 3 }; + +export const pause = (ms) => + new Promise((resolve) => + GLib.timeout_add(GLib.PRIORITY_DEFAULT, Math.max(0, ms), () => { + resolve(); + return GLib.SOURCE_REMOVE; + }) + ); + +export class Input { + constructor(capture) { + this.capture = capture; + } + + async pointer() { + const p = await shell.call('pointer'); + return { x: p.x, y: p.y }; + } + + async cursorInScreenshot() { + return this.capture.toPixel(await this.pointer()); + } + + async at(x, y) { + if (x !== undefined && x !== null && y !== undefined && y !== null) return this.capture.toPoint(x, y); + if ((x === undefined || x === null) && (y === undefined || y === null)) return this.pointer(); + throw new HelperError('Give both x and y.'); + } + + moveTo(p) { + return shell.call('motion', { x: p.x, y: p.y }); + } + + async move(x, y) { + await this.moveTo(await this.at(x, y)); + } + + async click(x, y, button = 'left', count = 1) { + const b = BUTTONS[button]; + if (!b) throw new HelperError('button must be left, right or middle.'); + await this.moveTo(await this.at(x, y)); + await pause(30); + const clicks = Math.max(1, Math.min(count, 3)); + for (let n = 1; n <= clicks; n++) { + await shell.call('button', { button: b, press: true }); + await pause(20); + await shell.call('button', { button: b, press: false }); + if (n < clicks) await pause(60); + } + } + + async drag(from, to) { + if (!from || !to || [from.x, from.y, to.x, to.y].some((n) => typeof n !== 'number')) throw new HelperError('A drag needs from {x,y} and to {x,y}.'); + const a = this.capture.toPoint(from.x, from.y); + const b = this.capture.toPoint(to.x, to.y); + await this.moveTo(a); + await pause(40); + await shell.call('button', { button: 1, press: true }); + await pause(80); + const steps = 12; + for (let i = 1; i <= steps; i++) { + const t = i / steps; + await this.moveTo({ x: a.x + (b.x - a.x) * t, y: a.y + (b.y - a.y) * t }); + await pause(20); + } + await pause(60); + await shell.call('button', { button: 1, press: false }); + } + + async scroll(x, y, dir, amount) { + if (!['up', 'down', 'left', 'right'].includes(dir)) throw new HelperError('scroll direction must be up, down, left or right.'); + await this.moveTo(await this.at(x, y)); + await pause(30); + await shell.call('scroll', { dir, steps: Math.max(1, Math.min(amount, 50)) }); + } + + async key(keysym, press) { + await shell.call('keyval', { keyval: keysym, press }); + } + + /** + * Plain ASCII goes as key presses, like a person typing. Anything else (č, + * emoji) may have no key on the layout, and the compositor drops a keysym it + * cannot place, so it goes through the input method instead, which takes + * any character. A field without input-method support (an X11 app) gets the + * key presses anyway. Returns a note when some characters may not have + * arrived. + */ + async type(text) { + if (!text) throw new HelperError('Nothing to type.'); + const runs = text.match(/[\x20-\x7e\n\r\t]+|[^\x20-\x7e\n\r\t]+/g) ?? []; + let unsure = ''; + for (const run of runs) { + if (/^[^\x20-\x7e\n\r\t]/.test(run)) { + const r = await shell.call('commit', { text: run }); + if (r.committed) { + await pause(30); + continue; + } + unsure += run; + } + for (const ch of run) { + const k = charKeysym(ch); + await this.key(k, true); + await this.key(k, false); + await pause(8); + } + } + return unsure + ? `The focused app takes no input-method text, so "${unsure}" went as key presses; characters missing from the keyboard layout are dropped. Check the screenshot.` + : null; + } + + async hold(combo, ms) { + const parsed = parseCombo(combo); + ms = Math.max(30, Math.min(ms || 0, 5000)); + try { + for (const m of parsed.modifiers) await this.key(m, true); + await this.key(parsed.keysym, true); + await pause(ms); + await this.key(parsed.keysym, false); + } finally { + for (const m of [...parsed.modifiers].reverse()) await this.key(m, false).catch(() => undefined); + } + } + + pressKey(combo) { + return this.hold(combo, 30); + } +} + +export { KEY }; diff --git a/native/linux/stem-computer/keys.js b/native/linux/stem-computer/keys.js new file mode 100644 index 00000000..e80543e5 --- /dev/null +++ b/native/linux/stem-computer/keys.js @@ -0,0 +1,68 @@ +// xdotool-style key names → X keysyms, which is what the compositor's virtual +// keyboard takes (it finds, or reserves, a key for each). Modifiers: ctrl, +// alt, shift, super. cmd/command mean Ctrl here: a model used to a Mac says +// cmd+c for copy, and Ctrl is what does that on Linux. + +import { HelperError } from './errors.js'; + +export const KEY = { + shift: 0xffe1, + ctrl: 0xffe3, + alt: 0xffe9, + super: 0xffeb, + return: 0xff0d, + tab: 0xff09 +}; + +const NAMED = { + return: 0xff0d, enter: 0xff0d, kp_enter: 0xff8d, tab: 0xff09, space: 0x20, + delete: 0xffff, backspace: 0xff08, escape: 0xff1b, esc: 0xff1b, + up: 0xff52, down: 0xff54, left: 0xff51, right: 0xff53, + home: 0xff50, end: 0xff57, page_up: 0xff55, pageup: 0xff55, page_down: 0xff56, pagedown: 0xff56, + insert: 0xff63, caps_lock: 0xffe5, menu: 0xff67, print: 0xff61, + minus: 0x2d, equal: 0x3d, plus: 0x2b, bracketleft: 0x5b, bracketright: 0x5d, + semicolon: 0x3b, apostrophe: 0x27, quote: 0x27, grave: 0x60, backslash: 0x5c, + comma: 0x2c, period: 0x2e, slash: 0x2f +}; +for (let i = 1; i <= 12; i++) NAMED[`f${i}`] = 0xffbd + i; + +export function modifier(name) { + switch (name.toLowerCase()) { + case 'ctrl': case 'control': case 'cmd': case 'command': return KEY.ctrl; + case 'alt': case 'option': case 'opt': return KEY.alt; + case 'shift': return KEY.shift; + case 'super': case 'win': case 'meta': case 'windows': return KEY.super; + default: return null; + } +} + +/** The keysym for one character, as X numbers them (Latin-1 as is, the rest at 0x1000000 + code point). */ +export function charKeysym(ch) { + const cp = ch.codePointAt(0); + if (ch === '\n' || ch === '\r') return KEY.return; + if (ch === '\t') return KEY.tab; + if ((cp >= 0x20 && cp <= 0x7e) || (cp >= 0xa0 && cp <= 0xff)) return cp; + return 0x1000000 + cp; +} + +/** "ctrl+shift+t" → { keysym, modifiers }. */ +export function parseCombo(combo) { + const parts = String(combo).split('+').map((p) => p.trim()); + const last = parts[parts.length - 1]; + if (!last) throw new HelperError('Name a key, e.g. "Return" or "ctrl+shift+t".'); + const modifiers = parts.slice(0, -1).map((p) => { + const m = modifier(p); + if (m === null) throw new HelperError(`Unknown modifier "${p}" (use ctrl, alt, shift, super).`); + return m; + }); + const named = NAMED[last.toLowerCase()]; + if (named !== undefined) return { keysym: named, modifiers }; + const lone = modifier(last); + if (lone !== null && modifiers.length === 0) return { keysym: lone, modifiers }; + if ([...last].length === 1) { + // A shortcut names the key, not the character: ctrl+C is ctrl+c. + const ch = modifiers.length > 0 ? last.toLowerCase() : last; + return { keysym: charKeysym(ch), modifiers }; + } + throw new HelperError(`Unknown key "${last}".`); +} diff --git a/native/linux/stem-computer/main.js b/native/linux/stem-computer/main.js new file mode 100644 index 00000000..1c515640 --- /dev/null +++ b/native/linux/stem-computer/main.js @@ -0,0 +1,479 @@ +// stem-computer for Linux: the GNOME Wayland half of Stem's computer control, +// the twin of native/mac/stem-computer and native/windows/stem-computer. A gjs +// child of the Stem desktop app that speaks one JSON object per line on +// stdin/stdout, with the same commands and replies as the other two, so +// src/desktop/computer-host drives any of them. Run as `gjs -m main.js`. +// +// Wayland lets no app look at other windows or move the pointer, so the +// screen, the window list and the virtual pointer and keyboard come from +// Stem's GNOME Shell extension (native/linux/gnome-extension, see shell.js); +// the controls inside a window come from AT-SPI (a11y.js), read here so a hung +// app never stalls the shell. +// +// Coordinates on the wire are pixels of the LAST SCREENSHOT this helper made +// (downscaled so the long side is at most 1568); the helper keeps the scale +// and maps them to stage coordinates itself. +// +// Two modes. SCREEN (the default): the frame is the primary monitor, input is +// the extension's virtual devices, and the person's first touch of their own +// mouse or keyboard ends the run. WINDOW (after select-window): the frame is +// one window wherever it sits in the stack, and clicks, typing and scrolling +// go through its accessibility tree, so it never comes forward — the run ends +// from the banner's Stop instead. + +import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; +import * as shell from './shell.js'; +import * as a11y from './a11y.js'; +import { Capture } from './capture.js'; +import { HelperError } from './errors.js'; +import { Input, pause } from './input.js'; +import { parseCombo } from './keys.js'; + +let GioUnix = null; +try { + GioUnix = (await import('gi://GioUnix')).default; +} catch { + // Older GLib: the Unix streams are still in Gio. +} + +const tracing = GLib.getenv('STEM_COMPUTER_TRACE') !== null; +const parentPid = Number.parseInt(GLib.getenv('STEM_PARENT_PID') ?? '', 10); +const loop = new GLib.MainLoop(null, false); +const stdout = new (GioUnix?.OutputStream ?? Gio.UnixOutputStream)({ fd: 1, close_fd: false }); +const stdin = new Gio.DataInputStream({ base_stream: new (GioUnix?.InputStream ?? Gio.UnixInputStream)({ fd: 0, close_fd: false }) }); + +const capture = new Capture(); +const input = new Input(capture); +let tree = null; +let wantShot = true; +let knownPid = null; +let knownWindows = null; + +function emit(obj) { + stdout.write_all(new TextEncoder().encode(`${JSON.stringify(obj)}\n`), null); + stdout.flush(null); +} + +function trace(message) { + if (tracing) printerr(`[stem-computer] ${message}`); +} + +const ACCESS_ALL = { screen: true, accessibility: true, inputMonitoring: true }; +const ACCESS_NONE = { screen: false, accessibility: false, inputMonitoring: false }; + +// ---- the kill switch ---------------------------------------------------- +// Starts disarmed, like the other helpers: looking never ends a run, and the +// first screen-mode input action arms it. +const watch = { + started: false, + armed: false, + unsubscribe: null, + async start() { + this.started = true; + this.unsubscribe ??= shell.onHumanInput((kind) => { + if (!this.armed) return; + this.armed = false; + emit({ event: 'human-input', kind }); + }); + }, + async arm() { + if (!this.started || this.armed) return; + // Armed before the input goes out, so the extension is listening when it lands. + await shell.call('watch', { on: true }); + this.armed = true; + }, + async disarm() { + this.armed = false; + if (this.started) await shell.call('watch', { on: false }).catch(() => undefined); + }, + async stop() { + await this.disarm(); + this.started = false; + this.unsubscribe?.(); + this.unsubscribe = null; + } +}; + +// ---- windows -------------------------------------------------------------- + +async function listWindows() { + const r = await shell.call('windows'); + // Stem's own windows are never listed. + return r.windows.filter((w) => w.pid !== parentPid && (w.minimized || (w.frame.width >= 50 && w.frame.height >= 50))); +} + +function brief(w) { + return `${w.id} ${w.title ? `"${w.title}"` : '(untitled)'} ${w.frame.width}x${w.frame.height}`; +} + +function describe(windows) { + if (windows.length === 0) return 'No windows are open.'; + const lines = ['Windows (id app "title" where size). select_window by window_id, or by app (+ title).']; + const order = [...windows].sort((a, b) => (b.focused ? 1 : 0) - (a.focused ? 1 : 0) || (a.minimized || a.otherWorkspace ? 1 : 0) - (b.minimized || b.otherWorkspace ? 1 : 0)); + for (const w of order) { + const place = w.minimized ? 'minimized' : w.otherWorkspace ? 'other workspace' : 'on screen (maybe covered)'; + const hint = w.minimized ? ' · minimized: cannot be pictured until restored' : ''; + lines.push(`${w.id} ${w.app} ${w.title ? `"${w.title}"` : '(untitled)'} ${place} ${w.frame.width}x${w.frame.height}${hint}${w.focused ? ' ← frontmost' : ''}`); + } + return lines.join('\n'); +} + +async function findWindow(windowId, app, title) { + const all = await listWindows(); + if (windowId !== undefined && windowId !== null) { + const hit = all.find((w) => w.id === windowId); + if (!hit) throw new HelperError(`No window ${windowId} is open now. Run list_windows again.`); + return hit; + } + if (!app) throw new HelperError('select_window needs window_id, or app (and optionally title).'); + const needle = app.toLowerCase(); + const names = (w) => [w.app, w.wmClass, w.appId?.replace(/\.desktop$/, '')].filter(Boolean).map((s) => s.toLowerCase()); + let cands = all.filter((w) => names(w).some((n) => n === needle || n.split('.').pop() === needle)); + if (cands.length === 0) cands = all.filter((w) => names(w).some((n) => n.includes(needle))); + if (cands.length === 0) { + const apps = [...new Set(all.map((w) => w.app))].sort(); + throw new HelperError(`No running app called "${app}" has a window. Apps with windows: ${apps.join(', ')}.`); + } + if (title) { + const t = title.toLowerCase(); + cands = cands.filter((w) => w.title.toLowerCase().includes(t)); + if (cands.length === 0) throw new HelperError(`${app} has no window whose title contains "${title}". Run list_windows to see its windows.`); + } + return cands.sort((a, b) => (a.minimized ? 1 : 0) - (b.minimized ? 1 : 0) || (a.otherWorkspace ? 1 : 0) - (b.otherWorkspace ? 1 : 0) || b.frame.width * b.frame.height - a.frame.width * a.frame.height)[0]; +} + +const windowMode = () => capture.target !== null; +const targetField = () => (capture.target ? { app: capture.target.app, title: capture.target.title, windowId: capture.target.id } : null); + +/** Refreshes the selected window's frame and title (it may have moved), for tree positions. */ +async function refreshTarget() { + if (!capture.target) return; + const w = (await listWindows()).find((x) => x.id === capture.target.id); + if (!w) return; + capture.target.title = w.title; + capture.target.frame = w.frame; + tree?.update(w); +} + +async function noteNewWindows() { + const t = capture.target; + if (!t) return null; + const now = (await listWindows()).filter((w) => w.pid === t.pid); + const before = knownWindows; + const samePid = knownPid === t.pid; + knownPid = t.pid; + knownWindows = new Set(now.map((w) => w.id)); + if (!before || !samePid) return null; + const fresh = now.filter((w) => !before.has(w.id) && w.id !== t.id); + if (fresh.length === 0) return null; + return `${t.app} opened a new window: ${fresh.map(brief).join('; ')}. It is a separate window, not in this one's picture: select_window window_id ${fresh[0].id} to work in it (staying in window mode), rather than switching to the whole screen.`; +} + +async function answerWithScreenshot(id, settleMs = 300, text = null) { + if (settleMs > 0) await pause(settleMs); + const said = [text, windowMode() ? await noteNewWindows() : null].filter(Boolean).join('\n\n'); + if (!wantShot) { + emit({ id, ok: true, text: said || 'Done.', target: targetField() }); + return; + } + const screenshot = await capture.screenshot(); + await refreshTarget(); + const reply = { id, ok: true, screenshot, cursor: await input.cursorInScreenshot(), target: targetField() }; + if (said) reply.text = said; + emit(reply); +} + +async function selectWindow(w) { + knownWindows = null; + if (w) { + if (!tree || tree.win.id !== w.id) tree = new a11y.WindowTree(w); + capture.target = { ...w }; + await watch.disarm(); + } else { + capture.target = null; + tree = null; + } +} + +const screenOnly = (what) => new HelperError(`${what} is a screen-mode action (it moves the real pointer). Clear the window first: select_window with no arguments.`); + +// ---- status ------------------------------------------------------------ + +async function status() { + const ext = await shell.extensionState(); + return { status: ext.state === 'ready' ? ACCESS_ALL : ACCESS_NONE, linux: ext }; +} + +// ---- Ask about this window ---------------------------------------------- + +async function peekWindow(id) { + const ext = await shell.extensionState(); + if (ext.state !== 'ready') { + emit({ id, ok: true, window: null, access: ACCESS_NONE, linux: ext }); + return; + } + const listed = (await listWindows()).filter((w) => !w.minimized && !w.otherWorkspace); + const w = listed.find((x) => x.focused) ?? listed[0]; + if (!w) { + emit({ id, ok: true, window: null, access: ACCESS_ALL }); + return; + } + const reply = { + id, + ok: true, + access: ACCESS_ALL, + window: { windowId: w.id, app: w.app, bundleId: w.appId?.replace(/\.desktop$/, '') ?? w.wmClass ?? null, title: w.title, bounds: w.frame } + }; + try { + const own = new Capture(); + own.target = { ...w }; + reply.screenshot = await own.screenshot(); + } catch (e) { + reply.text = e.message; + } + emit(reply); +} + +// ---- commands ------------------------------------------------------------ + +async function handle(obj) { + const id = obj.id ?? null; + const cmd = obj.cmd ?? ''; + wantShot = obj.shot !== false; + trace(`cmd ${cmd}`); + switch (cmd) { + case 'status': + case 'request-access': + emit({ id, ok: true, ...(await status()) }); + return; + case 'screenshot': + wantShot = true; + await answerWithScreenshot(id, 0); + return; + case 'cursor': + if (windowMode()) throw screenOnly('cursor_position'); + emit({ id, ok: true, cursor: await input.cursorInScreenshot(), target: targetField() }); + return; + case 'move': + if (windowMode()) throw screenOnly('mouse_move'); + await watch.arm(); + await input.move(obj.x, obj.y); + await answerWithScreenshot(id); + return; + case 'click': { + const button = obj.button ?? 'left'; + const count = obj.count ?? 1; + if (windowMode()) { + if (typeof obj.x !== 'number' || typeof obj.y !== 'number') throw new HelperError('In window mode a click needs a coordinate (there is no cursor to click at).'); + await refreshTarget(); + tree.click(capture.toPoint(obj.x, obj.y), button, count); + } else { + await watch.arm(); + await input.click(obj.x, obj.y, button, count); + } + await answerWithScreenshot(id); + return; + } + case 'drag': + if (windowMode()) throw screenOnly('left_click_drag'); + await watch.arm(); + await input.drag(obj.from, obj.to); + await answerWithScreenshot(id); + return; + case 'scroll': { + const dir = obj.dir ?? 'down'; + const amount = obj.amount ?? 3; + if (windowMode()) { + if (typeof obj.x !== 'number' || typeof obj.y !== 'number') throw new HelperError('In window mode scroll needs a coordinate over the area to scroll.'); + await refreshTarget(); + tree.scroll(capture.toPoint(obj.x, obj.y), dir, amount); + } else { + await watch.arm(); + await input.scroll(obj.x, obj.y, dir, amount); + } + await answerWithScreenshot(id); + return; + } + case 'type': + if (windowMode()) { + tree.type(obj.text ?? ''); + await answerWithScreenshot(id); + } else { + await watch.arm(); + await answerWithScreenshot(id, 300, await input.type(obj.text ?? '')); + } + return; + case 'key': + case 'hold': { + const combo = obj.combo ?? ''; + if (windowMode()) { + const parsed = parseCombo(combo); + const plainReturn = parsed.modifiers.length === 0 && (parsed.keysym === 0xff0d || parsed.keysym === 0xff8d); + if (!(plainReturn && tree.activateFocused())) { + throw new HelperError('On GNOME a background window takes no keys: Wayland routes the keyboard to the window in front only. Type text with `type`, act through snapshot and press, or clear the window and send the key in screen mode.'); + } + } else { + await watch.arm(); + if (cmd === 'key') await input.pressKey(combo); + else await input.hold(combo, obj.ms ?? 0); + } + await answerWithScreenshot(id); + return; + } + case 'wait': + await answerWithScreenshot(id, obj.ms ?? 1000); + return; + case 'zoom': + emit({ id, ok: true, screenshot: await capture.zoom(obj.x ?? 0, obj.y ?? 0, obj.w ?? 0, obj.h ?? 0), cursor: await input.cursorInScreenshot(), target: targetField() }); + return; + case 'list-windows': + emit({ id, ok: true, text: describe(await listWindows()), target: targetField() }); + return; + case 'select-window': { + const app = (obj.app ?? '').trim(); + const title = (obj.title ?? '').trim(); + if ((obj.windowId === undefined || obj.windowId === null) && !app) { + await selectWindow(null); + await answerWithScreenshot(id, 0); + return; + } + const w = await findWindow(obj.windowId, app, title); + await selectWindow(w); + await noteNewWindows(); + let screenshot; + try { + screenshot = await capture.screenshot(); + } catch (e) { + await selectWindow(null); + throw e; + } + const reply = { id, ok: true, target: targetField() }; + if (wantShot) { + reply.screenshot = screenshot; + reply.cursor = await input.cursorInScreenshot().catch(() => ({ x: 0, y: 0 })); + } else { + reply.text = 'Done.'; + } + emit(reply); + return; + } + case 'snapshot': { + if (!windowMode()) throw new HelperError('snapshot lists the controls of a selected window; select_window first (or take a screenshot of the screen).'); + const screenshot = await capture.screenshot(); + await refreshTarget(); + const depth = Math.max(1, Math.min(obj.depth ?? 12, 30)); + const text = tree.snapshot(depth, (p) => capture.toPixel(p), capture.unitsPerShotPixel); + const reply = { id, ok: true, text, target: targetField() }; + if (wantShot) { + reply.screenshot = screenshot; + reply.cursor = await input.cursorInScreenshot(); + } + emit(reply); + return; + } + case 'press': + case 'focus': + case 'menu': + case 'set-value': { + if (!windowMode()) throw new HelperError(`${cmd.replace('-', '_')} acts on a snapshot id; select_window and snapshot first.`); + const element = obj.element ?? -1; + if (cmd === 'press') tree.press(element); + else if (cmd === 'focus') tree.focus(element); + else if (cmd === 'menu') tree.menu(element); + else tree.setValue(element, obj.text ?? ''); + await answerWithScreenshot(id, cmd === 'focus' ? 100 : 300); + return; + } + case 'peek-window': + await peekWindow(id); + return; + case 'peek-text': { + const w = (await listWindows()).find((x) => x.id === obj.windowId); + let text = null; + if (w) { + try { + text = a11y.readableText(w, obj.maxChars ?? 20000); + } catch { + text = null; + } + } + emit({ id, ok: true, text }); + return; + } + case 'window-bounds': { + const w = (await listWindows()).find((x) => x.id === obj.windowId); + emit({ id, ok: true, bounds: w && !w.minimized ? w.frame : null }); + return; + } + case 'place-window': { + // Stem's own window (the run's banner) by title: Wayland gives Stem no + // way to place it, the extension does. It may not be mapped yet. + let w = null; + for (let tries = 0; !w && tries < 20; tries++) { + w = (await shell.call('windows')).windows.find((x) => x.pid === parentPid && x.title === obj.title) ?? null; + if (!w) await pause(100); + } + if (!w) throw new HelperError(`No window "${obj.title}" of Stem's is open.`); + await shell.call('place', { window: w.id, x: obj.x, y: obj.y }); + emit({ id, ok: true }); + return; + } + case 'reload-extension': + emit({ id, ok: true, protocol: await shell.reload() }); + return; + case 'watch': + if (obj.on ?? true) await watch.start(); + else await watch.stop(); + emit({ id, ok: true }); + return; + case 'stop': + await watch.stop(); + a11y.restoreToolkits(); + emit({ id, ok: true }); + loop.quit(); + return; + default: + emit({ id, ok: false, error: `unknown command "${cmd}"` }); + } +} + +// One command at a time, in order, like the other helpers. +let queue = Promise.resolve(); + +function readLine() { + stdin.read_line_async(GLib.PRIORITY_DEFAULT, null, (stream, result) => { + let line; + try { + [line] = stream.read_line_finish_utf8(result); + } catch { + line = null; + } + if (line === null) { + // Stem went away: stop like a `stop` would. + queue = queue.then(async () => { + await watch.stop().catch(() => undefined); + a11y.restoreToolkits(); + loop.quit(); + }); + return; + } + if (line.trim()) { + let obj = null; + try { + obj = JSON.parse(line); + } catch { + obj = null; + } + if (!obj || typeof obj !== 'object') emit({ ok: false, error: 'unreadable command' }); + else + queue = queue.then(() => + handle(obj).catch((e) => emit({ id: obj.id ?? null, ok: false, error: e instanceof HelperError ? e.message : `${e?.name ?? 'Error'}: ${e?.message ?? e}` })) + ); + } + readLine(); + }); +} + +readLine(); +loop.runAsync ? await loop.runAsync() : loop.run(); diff --git a/native/linux/stem-computer/shell.js b/native/linux/stem-computer/shell.js new file mode 100644 index 00000000..31b69897 --- /dev/null +++ b/native/linux/stem-computer/shell.js @@ -0,0 +1,146 @@ +// The GNOME Shell extension (native/linux/gnome-extension), as the helper +// reaches it: one D-Bus method on the shell's bus name, JSON in and out, the +// token read from the private folder the extension wrote it to. + +import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; +import { HelperError } from './errors.js'; + +export const UUID = 'stem-computer@awantech.sk'; +/** The extension protocol this helper speaks; an older running extension means "log out and back in". */ +export const PROTOCOL = 3; +const DEST = 'org.gnome.Shell'; +const PATH = '/sk/awantech/StemComputer'; +const IFACE = 'sk.awantech.StemComputer'; + +Gio._promisify(Gio.DBusConnection.prototype, 'call'); + +const bus = Gio.DBus.session; +let token = null; + +function tokenPath() { + return GLib.build_filenamev([GLib.get_user_runtime_dir(), 'stem-computer', 'token']); +} + +function readToken() { + try { + const [, bytes] = GLib.file_get_contents(tokenPath()); + return new TextDecoder().decode(bytes).trim(); + } catch { + return null; + } +} + +/** One request to the extension. Throws HelperError with the extension's own message. */ +export async function call(op, fields = {}, retried = false) { + token ??= readToken(); + if (!token) throw new HelperError(notReady()); + let reply; + try { + const result = await bus.call( + DEST, PATH, IFACE, 'Call', + new GLib.Variant('(ss)', [token, JSON.stringify({ ...fields, op })]), + new GLib.VariantType('(s)'), Gio.DBusCallFlags.NONE, 15000, null + ); + reply = JSON.parse(result.deepUnpack()[0]); + } catch (e) { + // The extension was re-enabled (new token), or is gone. + if (!retried && /Denied/.test(String(e?.message))) { + token = null; + return call(op, fields, true); + } + if (/UnknownMethod|ServiceUnknown|Object does not exist/.test(String(e?.message))) throw new HelperError(notReady()); + throw e; + } + if (!reply.ok) throw new HelperError(reply.error ?? `${op} failed`); + return reply; +} + +function notReady() { + return 'The Stem GNOME Shell extension is not running. Turn computer control on in Stem (Settings → Features → Computer control); after a first install, log out and back in.'; +} + +/** A `(u)` answer from the running extension, or 0 when it does not answer (or has no such method). */ +async function askNumber(method) { + try { + const r = await bus.call(DEST, PATH, IFACE, method, null, new GLib.VariantType('(u)'), Gio.DBusCallFlags.NONE, 3000, null); + return r.deepUnpack()[0]; + } catch { + return 0; + } +} + +const runningProtocol = () => askNumber('Version'); + +/** + * Have the running extension import its core.js again — the copy Stem just + * put on disk — instead of waiting for the next login. Needs the loader + * (LoaderVersion ≥ 1); an older extension has no Reload. Returns the protocol + * it now speaks. + */ +export async function reload() { + token ??= readToken(); + if (!token) throw new HelperError(notReady()); + try { + const r = await bus.call(DEST, PATH, IFACE, 'Reload', new GLib.Variant('(s)', [token]), new GLib.VariantType('(u)'), Gio.DBusCallFlags.NONE, 15000, null); + return r.deepUnpack()[0]; + } catch (e) { + throw new HelperError(`The GNOME extension could not reload: ${e?.message ?? e}`); + } +} + +/** + * Where the extension stands, in the words the desktop app shows: `ready`, + * `not-gnome`, `extensions-off` (the person turned user extensions off), + * `not-loaded` (installed or not, the shell has not loaded it: a fresh + * install loads at the next login), `disabled`, `error`, `outdated` (the shell + * runs an older copy than the helper needs, also cured by logging in again). + */ +export async function extensionState() { + const desktop = (GLib.getenv('XDG_CURRENT_DESKTOP') ?? '').split(':').map((s) => s.toLowerCase()); + if (!desktop.includes('gnome') && GLib.getenv('STEM_COMPUTER_ANY_DESKTOP') !== '1') return { state: 'not-gnome' }; + if (GLib.getenv('WAYLAND_DISPLAY') === null && GLib.getenv('XDG_SESSION_TYPE') !== 'wayland') return { state: 'not-wayland' }; + let info; + try { + const r = await bus.call( + DEST, '/org/gnome/Shell', 'org.gnome.Shell.Extensions', 'GetExtensionInfo', + new GLib.Variant('(s)', [UUID]), new GLib.VariantType('(a{sv})'), Gio.DBusCallFlags.NONE, 3000, null + ); + info = r.recursiveUnpack()[0]; + } catch { + return { state: 'not-gnome' }; + } + try { + const s = new Gio.Settings({ schema_id: 'org.gnome.shell' }); + if (s.get_boolean('disable-user-extensions')) return { state: 'extensions-off' }; + } catch { + // No shell schema: not a GNOME session after all. + } + if (!info || info.state === undefined) return { state: 'not-loaded' }; + // ExtensionState: 1 active, 2 inactive, 3 error, 4 out of date, 6 initialized (never enabled), 99 uninstalled. + switch (info.state) { + case 1: { + const protocol = await runningProtocol(); + // `loader`: whether an update can be reloaded in place (no login). + const loader = await askNumber('LoaderVersion'); + if (protocol < PROTOCOL) return { state: 'outdated', protocol, loader }; + return { state: 'ready', protocol, loader }; + } + case 3: + return { state: 'error', error: info.error ?? null }; + case 4: + return { state: 'error', error: 'This GNOME Shell version is newer than the extension knows.' }; + case 99: + return { state: 'not-loaded' }; + default: + return { state: 'disabled' }; + } +} + +/** Listen for the person's own input while a run is armed. Returns the unsubscribe. */ +export function onHumanInput(listener) { + const id = bus.signal_subscribe(DEST, IFACE, 'HumanInput', PATH, null, Gio.DBusSignalFlags.NONE, (_c, _s, _p, _i, _n, params) => + listener(params.deepUnpack()[0]) + ); + return () => bus.signal_unsubscribe(id); +} diff --git a/native/mac/stem-computer/AX.swift b/native/mac/stem-computer/AX.swift index b829558f..0303a050 100644 --- a/native/mac/stem-computer/AX.swift +++ b/native/mac/stem-computer/AX.swift @@ -92,7 +92,7 @@ final class AX { /// The AX window for our CGWindowID, giving an app that is still building /// its tree a moment (200 ms per try) before giving up. - private func lookUpWindow(tries: Int) -> AXUIElement? { + func lookUpWindow(tries: Int) -> AXUIElement? { for attempt in 0.. 0 { usleep(200_000) } var note = "" diff --git a/native/mac/stem-computer/WindowCapture.swift b/native/mac/stem-computer/WindowCapture.swift index 8fa43d8f..5fa963b8 100644 --- a/native/mac/stem-computer/WindowCapture.swift +++ b/native/mac/stem-computer/WindowCapture.swift @@ -18,15 +18,17 @@ enum WindowCapture { } /// Runs an async SCK call from the helper's synchronous command loop. - private static func wait(_ work: @escaping () async throws -> T) throws -> T { + private static func wait( + seconds: Double = 10, timeout message: String = "Capturing the window took too long.", _ work: @escaping () async throws -> T + ) throws -> T { let done = DispatchSemaphore(value: 0) var result: Result? Task.detached { do { result = .success(try await work()) } catch { result = .failure(error) } done.signal() } - if done.wait(timeout: .now() + 10) == .timedOut { - throw HelperError("Capturing the window took too long.") + if done.wait(timeout: .now() + seconds) == .timedOut { + throw HelperError(message) } switch result! { case .success(let v): return v @@ -44,7 +46,24 @@ enum WindowCapture { private static var cached: (windowID: CGWindowID, filter: SCContentFilter)? private static func filter(for windowID: CGWindowID) throws -> SCContentFilter { - try wait { + // On-screen windows first: a few dozen, where the full list carries every + // window the system holds. On 2026-10-14 CursorUIViewService had leaked + // 7,000 off-screen windows, the full list took 3 s inside the helper (10 s + // and a timeout under load), and the on-screen one 17 ms. + let onScreen = try wait { + try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + .windows.first(where: { $0.windowID == windowID }) + } + if let onScreen { return SCContentFilter(desktopIndependentWindow: onScreen) } + // A window on another Space or minimized is only in the full list. Give + // that its own, longer budget: a leak like the one above should make the + // lookup slow, not fail it, and when it does fail the model is told why + // and what still works. + trace("window \(windowID) is not on screen; looking it up in the full window list") + return try wait( + seconds: 30, + timeout: "macOS took too long to list its windows, and this window is not on the current screen (another Space, minimized or hidden), so only that list has it. Ask the user to bring the window onto the screen, or clear the window (select_window with no arguments) and use the whole screen, which does not need the list." + ) { let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) guard let window = content.windows.first(where: { $0.windowID == windowID }) else { throw HelperError("That window is gone. Run list_windows and select again.") diff --git a/native/mac/stem-computer/WindowPeek.swift b/native/mac/stem-computer/WindowPeek.swift new file mode 100644 index 00000000..f1f97732 --- /dev/null +++ b/native/mac/stem-computer/WindowPeek.swift @@ -0,0 +1,114 @@ +import AppKit +import ApplicationServices +import CoreGraphics +import Foundation + +// "Ask about this window": the desktop app's shortcut asks which window the +// person was in, takes one picture of it, and reads its text while they type. +// None of this touches a run's state — no window is selected, no capture +// geometry changes — so it can be served by a helper that is also driving a +// computer-control run, or by one of its own. + +enum WindowPeek { + /// The window the person was looking at: the front window of the frontmost + /// app, else (Stem itself was in front) the topmost ordinary window that is + /// not Stem's. Nil when no other app has a window on screen. + static func frontWindow() -> WindowInfo? { + guard let raw = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return nil } + // On-screen listing comes front to back; `Windows.list` has already dropped + // Stem's own windows, helpers and other non-Dock apps. + let known = Dictionary(Windows.list(minimized: false).map { ($0.id, $0) }, uniquingKeysWith: { a, _ in a }) + let ordered = raw.compactMap { ($0[kCGWindowNumber as String] as? Int).flatMap { known[CGWindowID($0)] } } + return ordered.first(where: { $0.frontmost }) ?? ordered.first + } + + static func describe(_ w: WindowInfo) -> [String: Any] { + [ + "windowId": Int(w.id), + "app": w.app, + "bundleId": w.bundleId ?? NSNull(), + "title": w.title, + "bounds": ["x": w.bounds.origin.x, "y": w.bounds.origin.y, "width": w.bounds.width, "height": w.bounds.height] + ] + } + + /// One frame of `w`, the same size and encoding a run's frames have. + static func picture(_ w: WindowInfo) throws -> [String: Any] { + let capture = Capture() + capture.target = Target(pid: w.pid, windowID: w.id, app: w.app, bundleId: w.bundleId, title: w.title, bounds: w.bounds) + return try capture.screenshot() + } + + /// The window's readable text, top to bottom, or nil when its app exposes none. + static func text(windowId: CGWindowID, maxChars: Int) -> String? { + guard let w = Windows.list(minimized: false).first(where: { $0.id == windowId }) else { return nil } + let ax = AX(pid: w.pid, windowID: w.id) + defer { ax.release() } + return ax.readableText(windowBounds: w.bounds, maxChars: maxChars) + } + + /// A window's current frame in global points, or nil once it is gone. + static func bounds(windowId: CGWindowID) -> [String: Any]? { + guard let b = Windows.bounds(of: windowId) else { return nil } + return ["x": b.origin.x, "y": b.origin.y, "width": b.width, "height": b.height] + } +} + +extension AX { + /// Every piece of text the window shows — labels, messages, field contents, + /// link text — in reading order, without the control ids and coordinates a + /// snapshot carries: this is context for a question, not a map to act on. + /// Secure fields are skipped; consecutive repeats are folded. + func readableText(windowBounds: CGRect, maxChars: Int) -> String? { + guard let root = window ?? lookUpWindow(tries: 5) else { return nil } + let visible = windowBounds.insetBy(dx: -2, dy: -2) + var parts: [String] = [] + var total = 0 + var visited = 0 + + func take(_ s: String?) { + guard let s = s?.trimmingCharacters(in: .whitespacesAndNewlines), !s.isEmpty, s != parts.last else { return } + parts.append(s) + total += s.count + 1 + } + + func walk(_ el: AXUIElement, depth: Int) { + if total >= maxChars || depth > 60 || visited > 8000 { return } + visited += 1 + if let f = AX.frame(el), f.width > 0, f.height > 0, !f.intersects(visible) { return } + let role = AX.string(el, kAXRoleAttribute as String) ?? "" + let subrole = AX.string(el, kAXSubroleAttribute as String) ?? "" + if subrole == "AXSecureTextField" { return } + switch role { + case "AXStaticText", "AXHeading": + take(AX.string(el, kAXValueAttribute as String) ?? AX.string(el, kAXTitleAttribute as String) ?? AX.string(el, kAXDescriptionAttribute as String)) + case "AXTextArea", "AXTextField", "AXComboBox": + take(AX.string(el, kAXValueAttribute as String)) + case "AXLink": + // A link's text is usually its child static text; only a bare link speaks for itself. + if AX.children(el).isEmpty { take(AX.string(el, kAXTitleAttribute as String) ?? AX.string(el, kAXDescriptionAttribute as String)) } + case "AXImage": + if let d = AX.string(el, kAXDescriptionAttribute as String), !d.isEmpty { take("[image: \(d)]") } + default: + break + } + for child in AX.children(el) { walk(child, depth: depth + 1) } + } + + walk(root, depth: 0) + // A Chromium window fills its tree over the first moments after + // accessibility is switched on (see AX.init): look again before believing + // an almost empty one. + var waited = 0 + while parts.count < 5 && waited < 6 { + usleep(400_000) + waited += 1 + parts = []; total = 0; visited = 0 + walk(root, depth: 0) + } + if parts.isEmpty { return nil } + var text = parts.joined(separator: "\n") + if text.count > maxChars { text = String(text.prefix(maxChars)) + "\n…" } + return text + } +} diff --git a/native/mac/stem-computer/main.swift b/native/mac/stem-computer/main.swift index 94568f02..674fb329 100644 --- a/native/mac/stem-computer/main.swift +++ b/native/mac/stem-computer/main.swift @@ -311,6 +311,29 @@ while let line = readLine(strippingNewline: true) { guard let ax = windowAX() else { throw HelperError("set_value acts on a snapshot id; select_window and snapshot first.") } try ax.setValue(id: Int(number(obj["element"]) ?? -1), text: obj["text"] as? String ?? "") answerWithScreenshot(id) + case "peek-window": + // "Ask about this window" (WindowPeek.swift): which window, and one picture of it. + let access = Status.check() + guard let w = WindowPeek.frontWindow() else { + emit(["id": id, "ok": true, "window": NSNull(), "access": access]) + break + } + var reply: [String: Any] = ["id": id, "ok": true, "window": WindowPeek.describe(w), "access": access] + if access["screen"] as? Bool == true { + do { reply["screenshot"] = try WindowPeek.picture(w) } catch { reply["text"] = "\(error)" } + } + emit(reply) + case "peek-text": + guard Status.check()["accessibility"] as? Bool == true else { + emit(["id": id, "ok": true, "text": NSNull()]) + break + } + let windowId = CGWindowID(Int(number(obj["windowId"]) ?? 0)) + let maxChars = Int(number(obj["maxChars"]) ?? 20_000) + emit(["id": id, "ok": true, "text": WindowPeek.text(windowId: windowId, maxChars: maxChars) ?? NSNull()]) + case "window-bounds": + let windowId = CGWindowID(Int(number(obj["windowId"]) ?? 0)) + emit(["id": id, "ok": true, "bounds": WindowPeek.bounds(windowId: windowId) ?? NSNull()]) case "watch": let on = obj["on"] as? Bool ?? true if on { diff --git a/native/windows/stem-computer/Capture.cs b/native/windows/stem-computer/Capture.cs new file mode 100644 index 00000000..66769e8f --- /dev/null +++ b/native/windows/stem-computer/Capture.cs @@ -0,0 +1,233 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Drawing2D; +using System.Drawing.Imaging; +using System.IO; +using System.Linq; + +namespace StemComputer +{ + class HelperError : Exception + { + public HelperError(string message) : base(message) { } + } + + /// The window a run works on once it has selected one. + class Target + { + public IntPtr Hwnd; + public uint Pid; + public string App; + public string Title; + public RECT Bounds; + + public Dictionary Summary() + { + return new Dictionary { { "app", App }, { "title", Title }, { "windowId", Hwnd.ToInt64() } }; + } + } + + /// Capture of the primary display, or of the selected window alone, + /// downscaled, plus the geometry that turns screenshot pixels back into + /// screen pixels. The same contract as the Mac helper's Capture.swift. + class Capture + { + /// Longest side of a frame handed to the model (Anthropic's computer-use guidance). + public const double MaxSide = 1568.0; + + public int LastWidth, LastHeight; + public double PixelsPerShotPixel = 1.0; + public int OriginX, OriginY; + + Target target; + public Target Target + { + get { return target; } + set + { + bool changed = (target == null) != (value == null) || (target != null && value != null && target.Hwnd != value.Hwnd); + target = value; + if (!changed) return; + LastWidth = LastHeight = 0; + PixelsPerShotPixel = 1.0; + OriginX = OriginY = 0; + } + } + + RECT CurrentBounds() + { + if (target == null) + { + return new RECT { Left = 0, Top = 0, Right = Native.GetSystemMetrics(Native.SM_CXSCREEN), Bottom = Native.GetSystemMetrics(Native.SM_CYSCREEN) }; + } + if (!Native.IsWindow(target.Hwnd)) + { + var left = Windows.AppWindows(target.Pid).Select(Windows.Brief).ToList(); + string others = left.Count == 0 ? "Run list_windows and select again." : target.App + "'s windows now: " + string.Join("; ", left) + " — select_window one of them."; + throw new HelperError("The selected window has closed (an action that closed it went through). " + others); + } + if (Native.IsIconic(target.Hwnd)) + { + throw new HelperError(target.App + "'s window is minimized, and Windows draws nothing for a minimized window. Ask the user to restore it (it can stay behind other windows), or clear the window and use the screen."); + } + target.Bounds = Native.Frame(target.Hwnd); + target.Title = Native.WindowText(target.Hwnd); + return target.Bounds; + } + + Bitmap Grab(RECT bounds) + { + if (target != null) return GrabWindow(target.Hwnd, bounds); + var bmp = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format32bppRgb); + using (var g = Graphics.FromImage(bmp)) + { + g.CopyFromScreen(bounds.Left, bounds.Top, 0, 0, new Size(bounds.Width, bounds.Height), CopyPixelOperation.SourceCopy); + } + return bmp; + } + + /// One window wherever it is in the stack: PrintWindow with full content + /// (Chromium and DirectX surfaces included), cropped to the visible frame. + public static Bitmap GrabWindow(IntPtr hwnd, RECT frame) + { + RECT outer; + Native.GetWindowRect(hwnd, out outer); + int w = Math.Max(1, outer.Width), h = Math.Max(1, outer.Height); + IntPtr screen = Native.GetDC(IntPtr.Zero); + IntPtr mem = Native.CreateCompatibleDC(screen); + IntPtr bitmap = Native.CreateCompatibleBitmap(screen, w, h); + IntPtr old = Native.SelectObject(mem, bitmap); + try + { + if (!Native.PrintWindow(hwnd, mem, Native.PW_RENDERFULLCONTENT)) + { + throw new HelperError("Windows would not draw that window (PrintWindow failed). Clear the window and use the screen."); + } + Native.SelectObject(mem, old); + using (var whole = Image.FromHbitmap(bitmap)) + { + var crop = new Rectangle(frame.Left - outer.Left, frame.Top - outer.Top, frame.Width, frame.Height); + crop.Intersect(new Rectangle(0, 0, whole.Width, whole.Height)); + if (crop.Width < 1 || crop.Height < 1) crop = new Rectangle(0, 0, whole.Width, whole.Height); + var outBmp = new Bitmap(crop.Width, crop.Height, PixelFormat.Format32bppRgb); + using (var g = Graphics.FromImage(outBmp)) + { + g.DrawImage(whole, new Rectangle(0, 0, crop.Width, crop.Height), crop, GraphicsUnit.Pixel); + } + return outBmp; + } + } + finally + { + Native.SelectObject(mem, old); + Native.DeleteObject(bitmap); + Native.DeleteDC(mem); + Native.ReleaseDC(IntPtr.Zero, screen); + } + } + + public Dictionary Screenshot() + { + var bounds = CurrentBounds(); + using (var image = Grab(bounds)) + { + double longest = Math.Max(image.Width, image.Height); + double factor = longest > MaxSide ? MaxSide / longest : 1.0; + int width = Math.Max(1, (int)Math.Round(image.Width * factor)); + int height = Math.Max(1, (int)Math.Round(image.Height * factor)); + LastWidth = width; + LastHeight = height; + PixelsPerShotPixel = bounds.Width / (double)width; + OriginX = bounds.Left; + OriginY = bounds.Top; + return new Dictionary { + { "jpegBase64", Jpeg(image, width, height) }, + { "width", width }, + { "height", height }, + { "scale", PixelsPerShotPixel } + }; + } + } + + public Dictionary Zoom(double x, double y, double w, double h) + { + if (LastWidth <= 0 || LastHeight <= 0) throw new HelperError("Take a screenshot before zooming."); + if (w < 1 || h < 1) throw new HelperError("The zoom region needs a positive width and height."); + var bounds = CurrentBounds(); + using (var image = Grab(bounds)) + { + double toFull = image.Width / (double)LastWidth; + var rect = new Rectangle((int)Math.Max(0, x * toFull), (int)Math.Max(0, y * toFull), (int)Math.Round(w * toFull), (int)Math.Round(h * toFull)); + rect.Intersect(new Rectangle(0, 0, image.Width, image.Height)); + if (rect.Width < 1 || rect.Height < 1) throw new HelperError("The zoom region lies outside the screen."); + using (var cropped = image.Clone(rect, image.PixelFormat)) + { + double factor = MaxSide / Math.Max(cropped.Width, cropped.Height); + int width = Math.Max(1, (int)Math.Round(cropped.Width * factor)); + int height = Math.Max(1, (int)Math.Round(cropped.Height * factor)); + return new Dictionary { + { "jpegBase64", Jpeg(cropped, width, height) }, + { "width", width }, + { "height", height }, + { "zoomed", true } + }; + } + } + } + + static ImageCodecInfo jpegCodec; + + public static string Jpeg(Image image, int width, int height) + { + Image scaled = image; + if (image.Width != width || image.Height != height) + { + var bmp = new Bitmap(width, height, PixelFormat.Format24bppRgb); + using (var g = Graphics.FromImage(bmp)) + { + g.InterpolationMode = InterpolationMode.HighQualityBicubic; + g.DrawImage(image, 0, 0, width, height); + } + scaled = bmp; + } + try + { + if (jpegCodec == null) jpegCodec = ImageCodecInfo.GetImageEncoders().First(c => c.MimeType == "image/jpeg"); + var parameters = new EncoderParameters(1); + parameters.Param[0] = new EncoderParameter(Encoder.Quality, 80L); + using (var ms = new MemoryStream()) + { + scaled.Save(ms, jpegCodec, parameters); + return Convert.ToBase64String(ms.ToArray()); + } + } + finally + { + if (!ReferenceEquals(scaled, image)) scaled.Dispose(); + } + } + + /// Screenshot pixels → screen pixels. + public POINT ToPoint(double x, double y) + { + if (LastWidth <= 0) throw new HelperError("Take a screenshot before pointing at it."); + double px = Math.Min(Math.Max(x, 0), LastWidth - 1); + double py = Math.Min(Math.Max(y, 0), LastHeight - 1); + return new POINT { X = OriginX + (int)Math.Round(px * PixelsPerShotPixel), Y = OriginY + (int)Math.Round(py * PixelsPerShotPixel) }; + } + + /// Screen pixels → screenshot pixels. + public int[] Pixel(int x, int y) + { + if (LastWidth <= 0) return new[] { x, y }; + return new[] { (int)Math.Round((x - OriginX) / PixelsPerShotPixel), (int)Math.Round((y - OriginY) / PixelsPerShotPixel) }; + } + + public Dictionary ToPixel(POINT p) + { + var xy = Pixel(p.X, p.Y); + return new Dictionary { { "x", xy[0] }, { "y", xy[1] } }; + } + } +} diff --git a/native/windows/stem-computer/Input.cs b/native/windows/stem-computer/Input.cs new file mode 100644 index 00000000..f16b9602 --- /dev/null +++ b/native/windows/stem-computer/Input.cs @@ -0,0 +1,314 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading; + +namespace StemComputer +{ + /// xdotool-style key names → Windows virtual keys. Modifiers: ctrl, alt, + /// shift, win (super/meta). cmd/command mean Ctrl here: a model used to a + /// Mac says cmd+c for copy, and Ctrl is what does that on Windows. + static class KeyMap + { + public class Parsed + { + public ushort Vk; + public bool Extended; + public List Modifiers = new List(); + } + + public const ushort VK_SHIFT = 0x10, VK_CONTROL = 0x11, VK_MENU = 0x12, VK_LWIN = 0x5B, VK_RETURN = 0x0D, VK_TAB = 0x09; + + static readonly Dictionary named = new Dictionary { + { "return", 0x0D }, { "enter", 0x0D }, { "kp_enter", 0x0D }, { "tab", 0x09 }, { "space", 0x20 }, + { "delete", 0x2E }, { "backspace", 0x08 }, { "escape", 0x1B }, { "esc", 0x1B }, + { "up", 0x26 }, { "down", 0x28 }, { "left", 0x25 }, { "right", 0x27 }, + { "home", 0x24 }, { "end", 0x23 }, { "page_up", 0x21 }, { "pageup", 0x21 }, { "page_down", 0x22 }, { "pagedown", 0x22 }, + { "insert", 0x2D }, { "caps_lock", 0x14 }, + { "f1", 0x70 }, { "f2", 0x71 }, { "f3", 0x72 }, { "f4", 0x73 }, { "f5", 0x74 }, { "f6", 0x75 }, + { "f7", 0x76 }, { "f8", 0x77 }, { "f9", 0x78 }, { "f10", 0x79 }, { "f11", 0x7A }, { "f12", 0x7B }, + { "minus", 0xBD }, { "equal", 0xBB }, { "plus", 0xBB }, { "bracketleft", 0xDB }, { "bracketright", 0xDD }, + { "semicolon", 0xBA }, { "apostrophe", 0xDE }, { "quote", 0xDE }, { "grave", 0xC0 }, { "backslash", 0xDC }, + { "comma", 0xBC }, { "period", 0xBE }, { "slash", 0xBF } + }; + + /// Keys whose scan code needs the extended flag, or they act as the numpad. + static readonly HashSet extended = new HashSet { 0x2E, 0x2D, 0x24, 0x23, 0x21, 0x22, 0x25, 0x26, 0x27, 0x28 }; + + public static ushort? Modifier(string name) + { + switch (name.ToLowerInvariant()) + { + case "ctrl": case "control": case "cmd": case "command": return VK_CONTROL; + case "alt": case "option": case "opt": return VK_MENU; + case "shift": return VK_SHIFT; + case "super": case "win": case "meta": case "windows": return VK_LWIN; + default: return null; + } + } + + public static Parsed Parse(string combo) + { + var parts = combo.Split('+'); + for (int i = 0; i < parts.Length; i++) parts[i] = parts[i].Trim(); + string last = parts[parts.Length - 1]; + if (last.Length == 0) throw new HelperError("Name a key, e.g. \"Return\" or \"ctrl+shift+t\"."); + var parsed = new Parsed(); + for (int i = 0; i < parts.Length - 1; i++) + { + var m = Modifier(parts[i]); + if (!m.HasValue) throw new HelperError("Unknown modifier \"" + parts[i] + "\" (use ctrl, alt, shift, win)."); + parsed.Modifiers.Add(m.Value); + } + string name = last.ToLowerInvariant(); + ushort vk; + if (named.TryGetValue(name, out vk)) + { + parsed.Vk = vk; + parsed.Extended = extended.Contains(vk); + return parsed; + } + if (last.Length == 1) + { + char ch = last[0]; + if (ch >= 'a' && ch <= 'z') { parsed.Vk = (ushort)(0x41 + (ch - 'a')); return parsed; } + if (ch >= 'A' && ch <= 'Z') { parsed.Vk = (ushort)(0x41 + (ch - 'A')); return parsed; } + if (ch >= '0' && ch <= '9') { parsed.Vk = (ushort)(0x30 + (ch - '0')); return parsed; } + short scan = Native.VkKeyScan(ch); + if (scan != -1) + { + parsed.Vk = (ushort)(scan & 0xFF); + if ((scan & 0x100) != 0) parsed.Modifiers.Add(VK_SHIFT); + if ((scan & 0x200) != 0) parsed.Modifiers.Add(VK_CONTROL); + if ((scan & 0x400) != 0) parsed.Modifiers.Add(VK_MENU); + return parsed; + } + } + var alone = Modifier(name); + if (alone.HasValue) { parsed.Vk = alone.Value; return parsed; } + throw new HelperError("Unknown key \"" + last + "\". Use xdotool names: Return, Tab, Escape, space, BackSpace, Delete, Up/Down/Left/Right, Home, End, Page_Up, Page_Down, F1–F12, or a single character."); + } + } + + /// Real input. Screen mode: SendInput, the person's own mouse and keyboard + /// (Windows marks these events injected, which is how the watch tells them + /// apart). Window mode: messages posted to the selected window, so it never + /// comes forward — which reaches classic Win32 controls and Chromium, but + /// not shortcuts that hold Ctrl or Alt (apps read those from the real + /// keyboard state). + class Input + { + readonly Capture capture; + /// Set in window mode: keys go to this window's focused control, not the session. + public IntPtr KeyboardWindow = IntPtr.Zero; + + public Input(Capture capture) + { + this.capture = capture; + } + + static void Pause(int ms) + { + Thread.Sleep(Math.Max(0, ms)); + } + + static void Send(params INPUT[] inputs) + { + uint sent = Native.SendInput((uint)inputs.Length, inputs, Marshal.SizeOf(typeof(INPUT))); + if (sent != inputs.Length) + { + throw new HelperError("Windows refused the input (another app may be running as administrator and in front, or the screen is locked)."); + } + } + + static INPUT Mouse(uint flags, int dx = 0, int dy = 0, int data = 0) + { + var i = new INPUT { type = Native.INPUT_MOUSE }; + i.u.mi = new MOUSEINPUT { dx = dx, dy = dy, mouseData = data, dwFlags = flags }; + return i; + } + + static INPUT Key(ushort vk, bool up, bool extendedKey = false) + { + var i = new INPUT { type = Native.INPUT_KEYBOARD }; + uint flags = (up ? Native.KEYEVENTF_KEYUP : 0) | (extendedKey ? Native.KEYEVENTF_EXTENDEDKEY : 0); + i.u.ki = new KEYBDINPUT { wVk = vk, wScan = (ushort)Native.MapVirtualKey(vk, Native.MAPVK_VK_TO_VSC), dwFlags = flags }; + return i; + } + + static INPUT Unicode(char unit, bool up) + { + var i = new INPUT { type = Native.INPUT_KEYBOARD }; + i.u.ki = new KEYBDINPUT { wVk = 0, wScan = unit, dwFlags = Native.KEYEVENTF_UNICODE | (up ? Native.KEYEVENTF_KEYUP : 0) }; + return i; + } + + public POINT CurrentPoint() + { + POINT p; + Native.GetCursorPos(out p); + return p; + } + + public Dictionary CursorInScreenshot() + { + return capture.ToPixel(CurrentPoint()); + } + + POINT At(double? x, double? y) + { + if (x.HasValue && y.HasValue) return capture.ToPoint(x.Value, y.Value); + if (!x.HasValue && !y.HasValue) return CurrentPoint(); + throw new HelperError("Give both x and y."); + } + + /// A move to screen pixel p, as absolute coordinates over the whole virtual desktop. + static INPUT MoveTo(POINT p) + { + int vx = Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), vy = Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN); + int vw = Math.Max(1, Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN) - 1), vh = Math.Max(1, Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN) - 1); + int nx = (int)Math.Round((p.X - vx) * 65535.0 / vw), ny = (int)Math.Round((p.Y - vy) * 65535.0 / vh); + return Mouse(Native.MOUSEEVENTF_MOVE | Native.MOUSEEVENTF_ABSOLUTE | Native.MOUSEEVENTF_VIRTUALDESK, nx, ny); + } + + public void Move(double? x, double? y) + { + Send(MoveTo(At(x, y))); + } + + public void Click(double? x, double? y, string button, int count) + { + var p = At(x, y); + uint down = Native.MOUSEEVENTF_LEFTDOWN, up = Native.MOUSEEVENTF_LEFTUP; + if (button == "right") { down = Native.MOUSEEVENTF_RIGHTDOWN; up = Native.MOUSEEVENTF_RIGHTUP; } + else if (button == "middle") { down = Native.MOUSEEVENTF_MIDDLEDOWN; up = Native.MOUSEEVENTF_MIDDLEUP; } + Send(MoveTo(p)); + Pause(30); + int clicks = Math.Max(1, Math.Min(count, 3)); + for (int n = 1; n <= clicks; n++) + { + Send(Mouse(down)); + Pause(20); + Send(Mouse(up)); + if (n < clicks) Pause(60); + } + } + + public void Drag(double? fx, double? fy, double? tx, double? ty) + { + if (!fx.HasValue || !fy.HasValue || !tx.HasValue || !ty.HasValue) throw new HelperError("A drag needs from {x,y} and to {x,y}."); + var a = capture.ToPoint(fx.Value, fy.Value); + var b = capture.ToPoint(tx.Value, ty.Value); + Send(MoveTo(a)); + Pause(40); + Send(Mouse(Native.MOUSEEVENTF_LEFTDOWN)); + Pause(80); + const int steps = 12; + for (int i = 1; i <= steps; i++) + { + double t = i / (double)steps; + Send(MoveTo(new POINT { X = (int)(a.X + (b.X - a.X) * t), Y = (int)(a.Y + (b.Y - a.Y) * t) })); + Pause(20); + } + Pause(60); + Send(Mouse(Native.MOUSEEVENTF_LEFTUP)); + } + + public void Scroll(double? x, double? y, string direction, int amount) + { + var p = At(x, y); + Send(MoveTo(p)); + Pause(30); + int lines = Math.Max(1, Math.Min(amount, 50)); + switch (direction) + { + case "up": Send(Mouse(Native.MOUSEEVENTF_WHEEL, 0, 0, 120 * lines)); break; + case "down": Send(Mouse(Native.MOUSEEVENTF_WHEEL, 0, 0, -120 * lines)); break; + case "left": Send(Mouse(Native.MOUSEEVENTF_HWHEEL, 0, 0, -120 * lines)); break; + case "right": Send(Mouse(Native.MOUSEEVENTF_HWHEEL, 0, 0, 120 * lines)); break; + default: throw new HelperError("scroll direction must be up, down, left or right."); + } + } + + /// The control that has keyboard focus inside the window-mode window (its + /// thread's focus), or the window itself. + IntPtr FocusOf(IntPtr hwnd) + { + uint pid; + uint thread = Native.GetWindowThreadProcessId(hwnd, out pid); + var info = new GUITHREADINFO { cbSize = Marshal.SizeOf(typeof(GUITHREADINFO)) }; + if (Native.GetGUIThreadInfo(thread, ref info) && info.hwndFocus != IntPtr.Zero) return info.hwndFocus; + return hwnd; + } + + static IntPtr KeyLParam(ushort vk, bool up, bool extendedKey) + { + uint scan = Native.MapVirtualKey(vk, Native.MAPVK_VK_TO_VSC); + long l = 1 | (scan << 16) | (extendedKey ? 1L << 24 : 0); + if (up) l |= (1L << 30) | (1L << 31); + return (IntPtr)l; + } + + public void Type(string text) + { + if (string.IsNullOrEmpty(text)) throw new HelperError("Nothing to type."); + if (KeyboardWindow != IntPtr.Zero) + { + var focus = FocusOf(KeyboardWindow); + for (int i = 0; i < text.Length; i++) + { + char unit = text[i]; + if (unit == '\r' && i + 1 < text.Length && text[i + 1] == '\n') continue; + if (unit == '\n' || unit == '\r' || unit == '\t') + { + // A key press, not WM_CHAR: rich edit controls (Windows 11 + // Notepad) drop a posted '\r' character but take the key, + // and the app's own TranslateMessage still makes the + // character for a plain edit control. + ushort vk = unit == '\t' ? KeyMap.VK_TAB : KeyMap.VK_RETURN; + Native.PostMessage(focus, Native.WM_KEYDOWN, (IntPtr)vk, KeyLParam(vk, false, false)); + Native.PostMessage(focus, Native.WM_KEYUP, (IntPtr)vk, KeyLParam(vk, true, false)); + } + else Native.PostMessage(focus, Native.WM_CHAR, (IntPtr)unit, (IntPtr)1); + Pause(4); + } + return; + } + foreach (char unit in text) + { + if (unit == '\n' || unit == '\r') Send(Key(KeyMap.VK_RETURN, false), Key(KeyMap.VK_RETURN, true)); + else if (unit == '\t') Send(Key(KeyMap.VK_TAB, false), Key(KeyMap.VK_TAB, true)); + else Send(Unicode(unit, false), Unicode(unit, true)); + Pause(8); + } + } + + public void PressKey(string combo) + { + Hold(combo, 30); + } + + public void Hold(string combo, int ms) + { + var parsed = KeyMap.Parse(combo); + ms = Math.Max(30, Math.Min(ms, 5000)); + if (KeyboardWindow != IntPtr.Zero) + { + if (parsed.Modifiers.Count > 0) + { + throw new HelperError("In window mode on Windows a background window only gets plain keys: anything with ctrl, alt, shift or win (shift+arrow selection too) needs the real keyboard. Use the menu through snapshot/press, or clear the window and send the shortcut in screen mode."); + } + var focus = FocusOf(KeyboardWindow); + Native.PostMessage(focus, Native.WM_KEYDOWN, (IntPtr)parsed.Vk, KeyLParam(parsed.Vk, false, parsed.Extended)); + Pause(ms); + Native.PostMessage(focus, Native.WM_KEYUP, (IntPtr)parsed.Vk, KeyLParam(parsed.Vk, true, parsed.Extended)); + return; + } + foreach (var m in parsed.Modifiers) Send(Key(m, false)); + Send(Key(parsed.Vk, false, parsed.Extended)); + Pause(ms); + Send(Key(parsed.Vk, true, parsed.Extended)); + for (int i = parsed.Modifiers.Count - 1; i >= 0; i--) Send(Key(parsed.Modifiers[i], true)); + } + } +} diff --git a/native/windows/stem-computer/Main.cs b/native/windows/stem-computer/Main.cs new file mode 100644 index 00000000..039b95b8 --- /dev/null +++ b/native/windows/stem-computer/Main.cs @@ -0,0 +1,419 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Web.Script.Serialization; + +// stem-computer.exe — the Windows half of Stem's computer control, the twin of +// native/mac/stem-computer. A child of the Stem desktop app that speaks one +// JSON object per line on stdin/stdout, with the same commands and replies as +// the Mac helper (see its main.swift), so src/desktop/computer-host drives +// either one. Windows asks for no permissions: capture, input and UI +// Automation are open to any app in the user's session. +// +// Coordinates on the wire are pixels of the LAST SCREENSHOT this helper made +// (downscaled so the long side is at most 1568); the helper keeps the scale +// and maps them to screen pixels itself. The process is per-monitor DPI aware, +// so every Windows API here speaks physical pixels. +// +// Two modes. SCREEN (the default): the frame is the primary display, input is +// real (SendInput), and the person's first touch of their own mouse or +// keyboard ends the run. WINDOW (after select-window): the frame is one window +// wherever it sits in the stack (PrintWindow), clicks go through UI Automation +// or as messages posted to that window, so it never comes forward — the run +// ends from the banner's Stop instead. +// +// Built with the C# compiler every Windows has (.NET Framework 4.8's csc.exe, +// C# 5) by scripts/build-windows-helper.mjs; no SDK needed. +namespace StemComputer +{ + static class Program + { + static readonly object outLock = new object(); + static readonly JavaScriptSerializer json = new JavaScriptSerializer { MaxJsonLength = int.MaxValue, RecursionLimit = 64 }; + static Stream stdout; + static readonly bool tracing = Environment.GetEnvironmentVariable("STEM_COMPUTER_TRACE") != null; + + static Capture capture; + static Input input; + static Watch watch; + static Uia uia; + static WindowInfo selected; + static bool wantShot = true; + /// The selected app's windows as last seen, to tell when an action opened another. + static uint knownPid; + static HashSet knownWindows; + + public static void Emit(Dictionary obj) + { + byte[] line = Encoding.UTF8.GetBytes(json.Serialize(obj) + "\n"); + lock (outLock) + { + stdout.Write(line, 0, line.Length); + stdout.Flush(); + } + } + + static void Trace(string message) + { + if (tracing) Console.Error.WriteLine("[stem-computer] " + message); + } + + static void Fail(object id, string message) + { + Emit(new Dictionary { { "id", id }, { "ok", false }, { "error", message } }); + } + + static double? Number(Dictionary obj, string key) + { + object v; + if (obj == null || !obj.TryGetValue(key, out v) || v == null) return null; + if (v is int) return (int)v; + if (v is long) return (long)v; + if (v is decimal) return (double)(decimal)v; + if (v is double) return (double)v; + return null; + } + + static string Str(Dictionary obj, string key) + { + object v; + return obj != null && obj.TryGetValue(key, out v) ? v as string : null; + } + + static bool? Bool(Dictionary obj, string key) + { + object v; + if (obj != null && obj.TryGetValue(key, out v) && v is bool) return (bool)v; + return null; + } + + static Dictionary Sub(Dictionary obj, string key) + { + object v; + return obj != null && obj.TryGetValue(key, out v) ? v as Dictionary : null; + } + + static bool WindowMode { get { return capture.Target != null; } } + + static object TargetField() + { + return capture.Target != null ? (object)capture.Target.Summary() : null; + } + + static Dictionary Access() + { + return new Dictionary { { "screen", true }, { "accessibility", true }, { "inputMonitoring", true } }; + } + + static string NoteNewWindows() + { + var t = capture.Target; + if (t == null) return null; + var now = Windows.AppWindows(t.Pid); + var before = knownWindows; + bool samePid = knownPid == t.Pid; + knownPid = t.Pid; + knownWindows = new HashSet(now.Select(w => w.Id)); + if (before == null || !samePid) return null; + var fresh = now.Where(w => !before.Contains(w.Id) && w.Hwnd != t.Hwnd).ToList(); + if (fresh.Count == 0) return null; + return t.App + " opened a new window: " + string.Join("; ", fresh.Select(Windows.Brief)) + ". It is a separate window, not in this one's picture: select_window window_id " + fresh[0].Id + " to work in it (staying in window mode), rather than switching to the whole screen."; + } + + static void AnswerWithScreenshot(object id, int settleMs = 300, string text = null) + { + if (settleMs > 0) Thread.Sleep(settleMs); + var said = string.Join("\n\n", new[] { text, WindowMode ? NoteNewWindows() : null }.Where(s => !string.IsNullOrEmpty(s))); + if (!wantShot) + { + Emit(new Dictionary { { "id", id }, { "ok", true }, { "text", said.Length == 0 ? "Done." : said }, { "target", TargetField() } }); + return; + } + var shot = capture.Screenshot(); + var reply = new Dictionary { { "id", id }, { "ok", true }, { "screenshot", shot }, { "cursor", input.CursorInScreenshot() }, { "target", TargetField() } }; + if (said.Length > 0) reply["text"] = said; + Emit(reply); + } + + static void SelectWindow(WindowInfo info) + { + selected = info; + knownWindows = null; + if (info != null) + { + if (uia == null || uia.Hwnd != info.Hwnd) uia = new Uia(info.Hwnd, info.Pid); + capture.Target = new Target { Hwnd = info.Hwnd, Pid = info.Pid, App = info.App, Title = info.Title, Bounds = info.Bounds }; + input.KeyboardWindow = info.Hwnd; + watch.Disarm(); + } + else + { + capture.Target = null; + input.KeyboardWindow = IntPtr.Zero; + uia = null; + } + } + + static HelperError ScreenOnly(string what) + { + return new HelperError(what + " is a screen-mode action (it moves the real mouse). Clear the window first: select_window with no arguments."); + } + + static void Handle(Dictionary obj) + { + object id = obj.ContainsKey("id") ? obj["id"] : null; + string cmd = Str(obj, "cmd") ?? ""; + wantShot = Bool(obj, "shot") ?? true; + Trace("cmd " + cmd); + try + { + switch (cmd) + { + case "status": + case "request-access": + Emit(new Dictionary { { "id", id }, { "ok", true }, { "status", Access() } }); + break; + case "screenshot": + wantShot = true; + AnswerWithScreenshot(id, 0); + break; + case "cursor": + if (WindowMode) throw ScreenOnly("cursor_position"); + Emit(new Dictionary { { "id", id }, { "ok", true }, { "cursor", input.CursorInScreenshot() }, { "target", TargetField() } }); + break; + case "move": + if (WindowMode) throw ScreenOnly("mouse_move"); + watch.Arm(); + input.Move(Number(obj, "x"), Number(obj, "y")); + AnswerWithScreenshot(id); + break; + case "click": + { + string button = Str(obj, "button") ?? "left"; + int count = (int)(Number(obj, "count") ?? 1); + if (WindowMode) + { + var x = Number(obj, "x"); + var y = Number(obj, "y"); + if (!x.HasValue || !y.HasValue) throw new HelperError("In window mode a click needs a coordinate (there is no cursor to click at)."); + uia.Click(capture.ToPoint(x.Value, y.Value), button, count); + } + else + { + watch.Arm(); + input.Click(Number(obj, "x"), Number(obj, "y"), button, count); + } + AnswerWithScreenshot(id); + break; + } + case "drag": + { + if (WindowMode) throw ScreenOnly("left_click_drag"); + watch.Arm(); + var from = Sub(obj, "from"); + var to = Sub(obj, "to"); + input.Drag(Number(from, "x"), Number(from, "y"), Number(to, "x"), Number(to, "y")); + AnswerWithScreenshot(id); + break; + } + case "scroll": + { + string dir = Str(obj, "dir") ?? "down"; + int amount = (int)(Number(obj, "amount") ?? 3); + if (WindowMode) + { + var x = Number(obj, "x"); + var y = Number(obj, "y"); + if (!x.HasValue || !y.HasValue) throw new HelperError("In window mode scroll needs a coordinate over the area to scroll."); + uia.Scroll(capture.ToPoint(x.Value, y.Value), dir, amount); + } + else + { + watch.Arm(); + input.Scroll(Number(obj, "x"), Number(obj, "y"), dir, amount); + } + AnswerWithScreenshot(id); + break; + } + case "type": + if (WindowMode && !uia.HasFocusedControl()) + { + throw new HelperError("Nothing in this window has keyboard focus. Click a field or `focus` its id first, then type."); + } + if (!WindowMode) watch.Arm(); + input.Type(Str(obj, "text") ?? ""); + AnswerWithScreenshot(id); + break; + case "key": + if (!WindowMode) watch.Arm(); + input.PressKey(Str(obj, "combo") ?? ""); + AnswerWithScreenshot(id); + break; + case "hold": + if (!WindowMode) watch.Arm(); + input.Hold(Str(obj, "combo") ?? "", (int)(Number(obj, "ms") ?? 0)); + AnswerWithScreenshot(id); + break; + case "wait": + AnswerWithScreenshot(id, (int)(Number(obj, "ms") ?? 1000)); + break; + case "zoom": + { + var shot = capture.Zoom(Number(obj, "x") ?? 0, Number(obj, "y") ?? 0, Number(obj, "w") ?? 0, Number(obj, "h") ?? 0); + Emit(new Dictionary { { "id", id }, { "ok", true }, { "screenshot", shot }, { "cursor", input.CursorInScreenshot() }, { "target", TargetField() } }); + break; + } + case "list-windows": + Emit(new Dictionary { { "id", id }, { "ok", true }, { "text", Windows.Describe(Windows.List()) }, { "target", TargetField() } }); + break; + case "select-window": + { + var windowId = Number(obj, "windowId"); + string app = (Str(obj, "app") ?? "").Trim(); + string title = (Str(obj, "title") ?? "").Trim(); + if (!windowId.HasValue && app.Length == 0) + { + SelectWindow(null); + AnswerWithScreenshot(id, 0); + break; + } + var info = Windows.Find(windowId.HasValue ? (long?)(long)windowId.Value : null, app, title); + SelectWindow(info); + NoteNewWindows(); + Dictionary shot; + try + { + shot = capture.Screenshot(); + } + catch (Exception) + { + SelectWindow(null); + throw; + } + var reply = new Dictionary { { "id", id }, { "ok", true }, { "target", TargetField() } }; + if (wantShot) + { + reply["screenshot"] = shot; + reply["cursor"] = input.CursorInScreenshot(); + } + else + { + reply["text"] = "Done."; + } + Emit(reply); + break; + } + case "snapshot": + { + if (!WindowMode) throw new HelperError("snapshot lists the controls of a selected window; select_window first (or take a screenshot of the screen)."); + var shot = capture.Screenshot(); + int depth = (int)(Number(obj, "depth") ?? 12); + string tree = uia.Snapshot(Math.Max(1, Math.Min(depth, 30)), capture.Target.Bounds, capture.Pixel, capture.PixelsPerShotPixel); + var reply = new Dictionary { { "id", id }, { "ok", true }, { "text", tree }, { "target", TargetField() } }; + if (wantShot) + { + reply["screenshot"] = shot; + reply["cursor"] = input.CursorInScreenshot(); + } + Emit(reply); + break; + } + case "press": + if (!WindowMode) throw new HelperError("press acts on a snapshot id; select_window and snapshot first."); + uia.Press((int)(Number(obj, "element") ?? -1)); + AnswerWithScreenshot(id); + break; + case "focus": + if (!WindowMode) throw new HelperError("focus acts on a snapshot id; select_window and snapshot first."); + uia.Focus((int)(Number(obj, "element") ?? -1)); + AnswerWithScreenshot(id, 100); + break; + case "menu": + if (!WindowMode) throw new HelperError("menu acts on a snapshot id; select_window and snapshot first."); + uia.Menu((int)(Number(obj, "element") ?? -1)); + AnswerWithScreenshot(id); + break; + case "set-value": + if (!WindowMode) throw new HelperError("set_value acts on a snapshot id; select_window and snapshot first."); + uia.SetValue((int)(Number(obj, "element") ?? -1), Str(obj, "text") ?? ""); + AnswerWithScreenshot(id); + break; + case "peek-window": + Peek.FrontWindow(id); + break; + case "peek-text": + Emit(new Dictionary { { "id", id }, { "ok", true }, { "text", Peek.Text((long)(Number(obj, "windowId") ?? 0), (int)(Number(obj, "maxChars") ?? 20000)) } }); + break; + case "window-bounds": + Emit(new Dictionary { { "id", id }, { "ok", true }, { "bounds", Peek.Bounds((long)(Number(obj, "windowId") ?? 0)) } }); + break; + case "watch": + if (Bool(obj, "on") ?? true) + { + watch.Start(kind => Emit(new Dictionary { { "event", "human-input" }, { "kind", kind } })); + } + else + { + watch.Stop(); + } + Emit(new Dictionary { { "id", id }, { "ok", true } }); + break; + case "stop": + watch.Stop(); + Emit(new Dictionary { { "id", id }, { "ok", true } }); + Environment.Exit(0); + break; + default: + Fail(id, "unknown command \"" + cmd + "\""); + break; + } + } + catch (HelperError e) + { + Fail(id, e.Message); + } + catch (Exception e) + { + Fail(id, e.GetType().Name + ": " + e.Message); + } + } + + [STAThread] + static void Main() + { + // Physical pixels everywhere (PER_MONITOR_AWARE_V2 = -4), or at least system-aware. + if (!Native.SetProcessDpiAwarenessContext((IntPtr)(-4))) Native.SetProcessDPIAware(); + uint parent; + if (uint.TryParse(Environment.GetEnvironmentVariable("STEM_PARENT_PID") ?? "", out parent)) Windows.ParentPid = parent; + stdout = Console.OpenStandardOutput(); + capture = new Capture(); + input = new Input(capture); + watch = new Watch(); + var reader = new StreamReader(Console.OpenStandardInput(), new UTF8Encoding(false)); + string line; + while ((line = reader.ReadLine()) != null) + { + if (line.Trim().Length == 0) continue; + Dictionary obj; + try + { + obj = json.DeserializeObject(line) as Dictionary; + } + catch (Exception) + { + obj = null; + } + if (obj == null) + { + Emit(new Dictionary { { "ok", false }, { "error", "unreadable command" } }); + continue; + } + Handle(obj); + } + watch.Stop(); + } + } +} diff --git a/native/windows/stem-computer/Native.cs b/native/windows/stem-computer/Native.cs new file mode 100644 index 00000000..a3880e20 --- /dev/null +++ b/native/windows/stem-computer/Native.cs @@ -0,0 +1,196 @@ +using System; +using System.Runtime.InteropServices; +using System.Text; + +// Win32 calls the helper needs. Everything works in physical pixels: the +// process declares itself per-monitor DPI aware at start (Main), so window +// rectangles, cursor positions and captures all share one unit. +namespace StemComputer +{ + [StructLayout(LayoutKind.Sequential)] + struct RECT + { + public int Left, Top, Right, Bottom; + public int Width { get { return Right - Left; } } + public int Height { get { return Bottom - Top; } } + } + + [StructLayout(LayoutKind.Sequential)] + struct POINT + { + public int X, Y; + } + + [StructLayout(LayoutKind.Sequential)] + struct MOUSEINPUT + { + public int dx, dy; + public int mouseData; + public uint dwFlags, time; + public IntPtr dwExtraInfo; + } + + [StructLayout(LayoutKind.Sequential)] + struct KEYBDINPUT + { + public ushort wVk, wScan; + public uint dwFlags, time; + public IntPtr dwExtraInfo; + } + + [StructLayout(LayoutKind.Explicit)] + struct InputUnion + { + [FieldOffset(0)] public MOUSEINPUT mi; + [FieldOffset(0)] public KEYBDINPUT ki; + } + + [StructLayout(LayoutKind.Sequential)] + struct INPUT + { + public uint type; + public InputUnion u; + } + + [StructLayout(LayoutKind.Sequential)] + struct MSLLHOOKSTRUCT + { + public POINT pt; + public uint mouseData, flags, time; + public IntPtr dwExtraInfo; + } + + [StructLayout(LayoutKind.Sequential)] + struct KBDLLHOOKSTRUCT + { + public uint vkCode, scanCode, flags, time; + public IntPtr dwExtraInfo; + } + + [StructLayout(LayoutKind.Sequential)] + struct GUITHREADINFO + { + public int cbSize; + public uint flags; + public IntPtr hwndActive, hwndFocus, hwndCapture, hwndMenuOwner, hwndMoveSize, hwndCaret; + public RECT rcCaret; + } + + [StructLayout(LayoutKind.Sequential)] + struct MSG + { + public IntPtr hwnd; + public uint message; + public IntPtr wParam, lParam; + public uint time; + public POINT pt; + } + + delegate bool EnumWindowsProc(IntPtr hwnd, IntPtr lParam); + delegate IntPtr HookProc(int code, IntPtr wParam, IntPtr lParam); + + static class Native + { + public const uint INPUT_MOUSE = 0, INPUT_KEYBOARD = 1; + public const uint MOUSEEVENTF_MOVE = 0x1, MOUSEEVENTF_LEFTDOWN = 0x2, MOUSEEVENTF_LEFTUP = 0x4, + MOUSEEVENTF_RIGHTDOWN = 0x8, MOUSEEVENTF_RIGHTUP = 0x10, MOUSEEVENTF_MIDDLEDOWN = 0x20, + MOUSEEVENTF_MIDDLEUP = 0x40, MOUSEEVENTF_WHEEL = 0x800, MOUSEEVENTF_HWHEEL = 0x1000, + MOUSEEVENTF_VIRTUALDESK = 0x4000, MOUSEEVENTF_ABSOLUTE = 0x8000; + public const uint KEYEVENTF_EXTENDEDKEY = 0x1, KEYEVENTF_KEYUP = 0x2, KEYEVENTF_UNICODE = 0x4; + public const int SM_XVIRTUALSCREEN = 76, SM_YVIRTUALSCREEN = 77, SM_CXVIRTUALSCREEN = 78, SM_CYVIRTUALSCREEN = 79, + SM_CXSCREEN = 0, SM_CYSCREEN = 1; + public const int WH_KEYBOARD_LL = 13, WH_MOUSE_LL = 14; + public const uint LLMHF_INJECTED = 0x1, LLMHF_LOWER_IL_INJECTED = 0x2, LLKHF_INJECTED = 0x10, LLKHF_LOWER_IL_INJECTED = 0x2; + public const int WM_MOUSEMOVE = 0x200, WM_LBUTTONDOWN = 0x201, WM_LBUTTONUP = 0x202, WM_LBUTTONDBLCLK = 0x203, + WM_RBUTTONDOWN = 0x204, WM_RBUTTONUP = 0x205, WM_MBUTTONDOWN = 0x207, WM_MBUTTONUP = 0x208, + WM_MOUSEWHEEL = 0x20A, WM_MOUSEHWHEEL = 0x20E, WM_KEYDOWN = 0x100, WM_KEYUP = 0x101, WM_CHAR = 0x102, + WM_CONTEXTMENU = 0x7B, WM_QUIT = 0x12; + public const int GWL_EXSTYLE = -20, WS_EX_TOOLWINDOW = 0x80, WS_EX_NOACTIVATE = 0x08000000; + public const uint GW_OWNER = 4, GW_HWNDNEXT = 2, GA_ROOT = 2; + public const int DWMWA_EXTENDED_FRAME_BOUNDS = 9, DWMWA_CLOAKED = 14; + public const uint PW_RENDERFULLCONTENT = 2; + public const uint SRCCOPY = 0x00CC0020, CAPTUREBLT = 0x40000000; + public const uint MAPVK_VK_TO_VSC = 0; + + [DllImport("user32.dll")] public static extern uint SendInput(uint n, INPUT[] inputs, int size); + [DllImport("user32.dll")] public static extern int GetSystemMetrics(int index); + [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p); + [DllImport("user32.dll")] public static extern bool EnumWindows(EnumWindowsProc proc, IntPtr lParam); + [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr hwnd); + [DllImport("user32.dll")] public static extern bool IsWindow(IntPtr hwnd); + [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr hwnd); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowText(IntPtr hwnd, StringBuilder text, int max); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextLength(IntPtr hwnd); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetClassName(IntPtr hwnd, StringBuilder name, int max); + [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid); + [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect); + [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hwnd, int index); + [DllImport("user32.dll")] public static extern IntPtr GetWindow(IntPtr hwnd, uint cmd); + [DllImport("user32.dll")] public static extern IntPtr GetAncestor(IntPtr hwnd, uint flags); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll")] public static extern IntPtr GetTopWindow(IntPtr hwnd); + [DllImport("user32.dll")] public static extern bool PrintWindow(IntPtr hwnd, IntPtr hdc, uint flags); + [DllImport("user32.dll")] public static extern IntPtr GetDC(IntPtr hwnd); + [DllImport("user32.dll")] public static extern int ReleaseDC(IntPtr hwnd, IntPtr hdc); + [DllImport("user32.dll")] public static extern bool PostMessage(IntPtr hwnd, int msg, IntPtr wParam, IntPtr lParam); + [DllImport("user32.dll")] public static extern bool ScreenToClient(IntPtr hwnd, ref POINT p); + [DllImport("user32.dll")] public static extern IntPtr ChildWindowFromPointEx(IntPtr parent, POINT p, uint flags); + [DllImport("user32.dll")] public static extern bool GetGUIThreadInfo(uint threadId, ref GUITHREADINFO info); + [DllImport("user32.dll")] public static extern short VkKeyScan(char ch); + [DllImport("user32.dll")] public static extern uint MapVirtualKey(uint code, uint mapType); + [DllImport("user32.dll")] public static extern IntPtr SetWindowsHookEx(int id, HookProc proc, IntPtr module, uint threadId); + [DllImport("user32.dll")] public static extern bool UnhookWindowsHookEx(IntPtr hook); + [DllImport("user32.dll")] public static extern IntPtr CallNextHookEx(IntPtr hook, int code, IntPtr wParam, IntPtr lParam); + [DllImport("user32.dll")] public static extern int GetMessage(out MSG msg, IntPtr hwnd, uint min, uint max); + [DllImport("user32.dll")] public static extern bool PostThreadMessage(uint threadId, int msg, IntPtr wParam, IntPtr lParam); + [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); + [DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); + [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] public static extern IntPtr GetModuleHandle(string name); + [DllImport("dwmapi.dll")] public static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out RECT value, int size); + [DllImport("dwmapi.dll")] public static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out int value, int size); + [DllImport("gdi32.dll")] public static extern IntPtr CreateCompatibleDC(IntPtr hdc); + [DllImport("gdi32.dll")] public static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int w, int h); + [DllImport("gdi32.dll")] public static extern IntPtr SelectObject(IntPtr hdc, IntPtr obj); + [DllImport("gdi32.dll")] public static extern bool DeleteObject(IntPtr obj); + [DllImport("gdi32.dll")] public static extern bool DeleteDC(IntPtr hdc); + [DllImport("gdi32.dll")] public static extern bool BitBlt(IntPtr dst, int x, int y, int w, int h, IntPtr src, int sx, int sy, uint rop); + + public static IntPtr MakeLParam(int lo, int hi) + { + return (IntPtr)((hi << 16) | (lo & 0xFFFF)); + } + + public static string WindowText(IntPtr hwnd) + { + int n = GetWindowTextLength(hwnd); + if (n <= 0) return ""; + var sb = new StringBuilder(n + 1); + GetWindowText(hwnd, sb, sb.Capacity); + return sb.ToString(); + } + + public static string ClassName(IntPtr hwnd) + { + var sb = new StringBuilder(256); + GetClassName(hwnd, sb, sb.Capacity); + return sb.ToString(); + } + + /// The window's visible frame: DWM's bounds, without the invisible resize border GetWindowRect includes. + public static RECT Frame(IntPtr hwnd) + { + RECT r; + if (DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, out r, Marshal.SizeOf(typeof(RECT))) == 0 && r.Width > 0) return r; + GetWindowRect(hwnd, out r); + return r; + } + + /// Hidden by the shell: on another virtual desktop, or a suspended Store app. + public static bool Cloaked(IntPtr hwnd) + { + int cloaked; + return DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, out cloaked, 4) == 0 && cloaked != 0; + } + } +} diff --git a/native/windows/stem-computer/Peek.cs b/native/windows/stem-computer/Peek.cs new file mode 100644 index 00000000..f586a0ae --- /dev/null +++ b/native/windows/stem-computer/Peek.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace StemComputer +{ + /// "Ask about this window": which window the person is in, one picture of + /// it, its readable text, and where it is now (the panel docks beside it). + /// The Windows twin of WindowPeek.swift. Bounds are physical pixels; the + /// desktop app converts them to its own units. + static class Peek + { + static Dictionary Rect(RECT r) + { + return new Dictionary { { "x", r.Left }, { "y", r.Top }, { "width", r.Width }, { "height", r.Height } }; + } + + /// The foreground window, or the next one down when Stem itself is in front. + static WindowInfo Front() + { + var listed = Windows.List().Where(w => !w.Minimized && !w.OtherDesktop).ToList(); + var fg = Native.GetForegroundWindow(); + var hit = listed.FirstOrDefault(w => w.Hwnd == fg); + return hit ?? listed.FirstOrDefault(); + } + + public static void FrontWindow(object id) + { + var access = new Dictionary { { "screen", true }, { "accessibility", true }, { "inputMonitoring", true } }; + var w = Front(); + if (w == null) + { + Program.Emit(new Dictionary { { "id", id }, { "ok", true }, { "window", null }, { "access", access } }); + return; + } + var reply = new Dictionary { + { "id", id }, { "ok", true }, { "access", access }, + { "window", new Dictionary { + { "windowId", w.Id }, { "app", w.App }, + // What MCP app matching keys on, like a Mac bundle id: the program's file name. + { "bundleId", string.IsNullOrEmpty(w.Exe) ? null : Path.GetFileNameWithoutExtension(w.Exe).ToLowerInvariant() }, + { "title", w.Title }, { "bounds", Rect(w.Bounds) } } } + }; + try + { + var capture = new Capture(); + capture.Target = new Target { Hwnd = w.Hwnd, Pid = w.Pid, App = w.App, Title = w.Title, Bounds = w.Bounds }; + reply["screenshot"] = capture.Screenshot(); + } + catch (Exception e) + { + reply["text"] = e.Message; + } + Program.Emit(reply); + } + + public static string Text(long windowId, int maxChars) + { + var w = Windows.List().FirstOrDefault(x => x.Id == windowId); + if (w == null) return null; + try + { + return new Uia(w.Hwnd, w.Pid).ReadableText(w.Bounds, maxChars); + } + catch (Exception) + { + return null; + } + } + + public static object Bounds(long windowId) + { + var hwnd = new IntPtr(windowId); + if (!Native.IsWindow(hwnd) || Native.IsIconic(hwnd)) return null; + return Rect(Native.Frame(hwnd)); + } + } +} diff --git a/native/windows/stem-computer/Uia.cs b/native/windows/stem-computer/Uia.cs new file mode 100644 index 00000000..2e679308 --- /dev/null +++ b/native/windows/stem-computer/Uia.cs @@ -0,0 +1,501 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Windows; +using System.Windows.Automation; + +namespace StemComputer +{ + // UI Automation: how a run reaches a window that is not in front — the + // Windows twin of the Mac helper's AX.swift. Where screen mode moves the + // real mouse, window mode asks the window's automation tree: press this + // button, set that field's value. Coverage follows the app's own tree: + // Win32, WinForms, WPF, UWP and Chromium answer; canvases and games are blank. + class Uia + { + public readonly IntPtr Hwnd; + public readonly uint Pid; + AutomationElement root; + readonly Dictionary nodes = new Dictionary(); + bool snapshotTaken; + public int LastSnapshotCount; + const int MaxNodes = 400; + + static readonly CacheRequest cache = MakeCache(); + + static CacheRequest MakeCache() + { + var cr = new CacheRequest(); + cr.AutomationElementMode = AutomationElementMode.Full; + foreach (var p in new AutomationProperty[] { + AutomationElement.NameProperty, AutomationElement.ControlTypeProperty, AutomationElement.BoundingRectangleProperty, + AutomationElement.IsEnabledProperty, AutomationElement.HasKeyboardFocusProperty, AutomationElement.IsOffscreenProperty, + AutomationElement.IsPasswordProperty, AutomationElement.HelpTextProperty, AutomationElement.NativeWindowHandleProperty, + AutomationElement.IsInvokePatternAvailableProperty, AutomationElement.IsTogglePatternAvailableProperty, + AutomationElement.IsSelectionItemPatternAvailableProperty, AutomationElement.IsExpandCollapsePatternAvailableProperty, + AutomationElement.IsValuePatternAvailableProperty, AutomationElement.IsScrollPatternAvailableProperty, + ValuePattern.ValueProperty, ValuePattern.IsReadOnlyProperty }) + { + cr.Add(p); + } + cr.TreeScope = TreeScope.Element; + return cr; + } + + public Uia(IntPtr hwnd, uint pid) + { + Hwnd = hwnd; + Pid = pid; + } + + AutomationElement Root() + { + if (root != null) return root; + try + { + root = AutomationElement.FromHandle(Hwnd); + } + catch (Exception e) + { + throw new HelperError("This window exposes no UI Automation tree (" + e.Message + "); clear the window (select_window with no arguments) and use the screen."); + } + return root; + } + + static List Children(AutomationElement el) + { + try + { + using (cache.Activate()) + { + return el.FindAll(TreeScope.Children, Condition.TrueCondition).Cast().ToList(); + } + } + catch (Exception) + { + // The element went away, or the app stopped answering. + return new List(); + } + } + + static T Cached(AutomationElement el, AutomationProperty p, T fallback) + { + try + { + object v = el.GetCachedPropertyValue(p, true); + if (v == AutomationElement.NotSupported || v == null) return fallback; + return (T)v; + } + catch (Exception) + { + try + { + object v = el.GetCurrentPropertyValue(p, true); + if (v == AutomationElement.NotSupported || v == null) return fallback; + return (T)v; + } + catch (Exception) + { + return fallback; + } + } + } + + static string Clip(string s, int limit = 80) + { + if (s == null) return null; + s = s.Replace("\r\n", "⏎").Replace("\n", "⏎").Trim(); + if (s.Length == 0) return null; + return s.Length > limit ? s.Substring(0, limit - 1) + "…" : s; + } + + static readonly Dictionary roleNames = new Dictionary { + { "Button", "button" }, { "SplitButton", "menubutton" }, { "RadioButton", "radio" }, { "CheckBox", "checkbox" }, + { "Edit", "textfield" }, { "Document", "document" }, { "Text", "text" }, { "Hyperlink", "link" }, { "Image", "image" }, + { "MenuItem", "menuitem" }, { "Menu", "menu" }, { "MenuBar", "menubar" }, { "List", "list" }, { "ListItem", "row" }, + { "DataGrid", "table" }, { "DataItem", "row" }, { "Table", "table" }, { "Tree", "outline" }, { "TreeItem", "row" }, + { "Tab", "tabs" }, { "TabItem", "tab" }, { "ToolBar", "toolbar" }, { "Slider", "slider" }, { "ComboBox", "combobox" }, + { "Group", "group" }, { "Pane", "pane" }, { "Window", "window" }, { "ScrollBar", "scrollbar" }, { "Header", "header" }, + { "HeaderItem", "column" }, { "StatusBar", "statusbar" }, { "ProgressBar", "progress" }, { "Spinner", "stepper" }, + { "TitleBar", "titlebar" }, { "Separator", "separator" }, { "Calendar", "calendar" }, { "Custom", "custom" }, + { "Thumb", "thumb" }, { "ToolTip", "tooltip" }, { "SemanticZoom", "zoom" }, { "AppBar", "toolbar" } + }; + + static readonly HashSet structural = new HashSet { "Group", "Pane", "Custom", "List", "Table", "DataGrid", "Tree", "ToolBar", "Tab", "Document", "Header", "TitleBar", "Separator", "ScrollBar", "Thumb" }; + + static string RoleOf(AutomationElement el) + { + var ct = Cached(el, AutomationElement.ControlTypeProperty, ControlType.Custom); + string name = ct.ProgrammaticName.Replace("ControlType.", ""); + return name; + } + + /// The window's controls, one line each with a fresh id, positioned in + /// pixels of the current window picture (`toPixel`). + public string Snapshot(int maxDepth, RECT window, Func toPixel, double pixelsPerShotPixel) + { + var top = Root(); + nodes.Clear(); + snapshotTaken = true; + var lines = new List(); + int next = 0; + bool truncated = false; + var visible = new Rect(window.Left - 2, window.Top - 2, window.Width + 4, window.Height + 4); + var clock = Stopwatch.StartNew(); + + Action walk = null; + walk = (el, depth, indent) => + { + if (nodes.Count >= MaxNodes || clock.ElapsedMilliseconds > 8000) { truncated = true; return; } + string role = RoleOf(el); + var frame = Cached(el, AutomationElement.BoundingRectangleProperty, Rect.Empty); + bool hasFrame = !frame.IsEmpty && frame.Width > 0 && frame.Height > 0; + if (hasFrame && !frame.IntersectsWith(visible)) return; + string label = Clip(Cached(el, AutomationElement.NameProperty, null)); + string help = Clip(Cached(el, AutomationElement.HelpTextProperty, null)); + bool password = Cached(el, AutomationElement.IsPasswordProperty, false); + bool valueAvailable = Cached(el, AutomationElement.IsValuePatternAvailableProperty, false); + string value = valueAvailable && !password ? Clip(Cached(el, ValuePattern.ValueProperty, null)) : null; + bool readOnly = Cached(el, ValuePattern.IsReadOnlyProperty, true); + bool focused = Cached(el, AutomationElement.HasKeyboardFocusProperty, false); + bool enabled = Cached(el, AutomationElement.IsEnabledProperty, true); + bool pressable = Cached(el, AutomationElement.IsInvokePatternAvailableProperty, false) + || Cached(el, AutomationElement.IsTogglePatternAvailableProperty, false) + || Cached(el, AutomationElement.IsSelectionItemPatternAvailableProperty, false) + || Cached(el, AutomationElement.IsExpandCollapsePatternAvailableProperty, false); + bool isStructure = structural.Contains(role) && label == null && !focused; + if (!isStructure && hasFrame && role != "Window") + { + int id = next++; + nodes[id] = el; + var caps = new List(); + if (pressable) caps.Add("press"); + caps.Add("menu"); + if (valueAvailable && !readOnly) caps.Add("setvalue"); + string roleName; + if (!roleNames.TryGetValue(role, out roleName)) roleName = role.ToLowerInvariant(); + if (password) roleName = "password"; + string line = new string(' ', Math.Min(indent, 12)) + id + " " + roleName; + if (label != null) line += " \"" + label + "\""; + else if (help != null) line += " \"" + help + "\""; + if (value != null && value != label && role != "Text") line += " = \"" + value + "\""; + if (focused) line += " focused"; + if (!enabled) line += " disabled"; + line += " [" + string.Join(",", caps) + "]"; + var xy = toPixel((int)frame.X, (int)frame.Y); + line += " (" + xy[0] + "," + xy[1] + " " + (int)Math.Round(frame.Width / pixelsPerShotPixel) + "x" + (int)Math.Round(frame.Height / pixelsPerShotPixel) + ")"; + lines.Add(line); + } + if (depth >= maxDepth) return; + foreach (var child in Children(el)) walk(child, depth + 1, isStructure ? indent : indent + 1); + }; + + using (cache.Activate()) + { + try { top = top.GetUpdatedCache(cache); } catch (Exception) { } + } + walk(top, 0, 0); + // A Chromium window builds its tree on the first question and fills + // it over a second or two: look again before believing a bare one. + int waited = 0; + while (lines.Count < 12 && !truncated && waited < 6) + { + Thread.Sleep(500); + waited++; + nodes.Clear(); lines.Clear(); next = 0; + walk(top, 0, 0); + } + LastSnapshotCount = lines.Count; + string head = "Controls of this window (id role \"label\" [what it can do] (x,y wxh in window pixels)). Ids are valid until the next snapshot."; + if (truncated) head += " Only the first " + nodes.Count + " controls are listed; lower `depth` or scroll to see others."; + if (lines.Count == 0) return head + "\n(none exposed — this window may be a canvas; act on it in screen mode)"; + return head + "\n" + string.Join("\n", lines); + } + + AutomationElement Node(int id) + { + if (!snapshotTaken) throw new HelperError("Take a snapshot first; element ids come from it."); + AutomationElement el; + if (!nodes.TryGetValue(id, out el)) throw new HelperError("No element " + id + " in the last snapshot. Take a new snapshot and use one of its ids."); + return el; + } + + static object Pattern(AutomationElement el, AutomationPattern p) + { + object pattern; + try + { + return el.TryGetCurrentPattern(p, out pattern) ? pattern : null; + } + catch (Exception) + { + return null; + } + } + + /// The element's own action (invoke, toggle, select, expand), or its nearest ancestor's. + bool Press(AutomationElement el) + { + var walker = TreeWalker.ControlViewWalker; + var at = el; + for (int i = 0; i < 4 && at != null; i++) + { + try + { + var invoke = Pattern(at, InvokePattern.Pattern) as InvokePattern; + if (invoke != null) { invoke.Invoke(); return true; } + var toggle = Pattern(at, TogglePattern.Pattern) as TogglePattern; + if (toggle != null) { toggle.Toggle(); return true; } + var select = Pattern(at, SelectionItemPattern.Pattern) as SelectionItemPattern; + if (select != null) { select.Select(); return true; } + var expand = Pattern(at, ExpandCollapsePattern.Pattern) as ExpandCollapsePattern; + if (expand != null) + { + if (expand.Current.ExpandCollapseState == ExpandCollapseState.Expanded) expand.Collapse(); else expand.Expand(); + return true; + } + } + catch (ElementNotAvailableException) + { + throw new HelperError("The control is gone; take a new snapshot."); + } + catch (InvalidOperationException) + { + // Disabled, or the app refused: try the parent. + } + if (at == Root()) break; + try { at = walker.GetParent(at); } catch (Exception) { break; } + } + return false; + } + + static Rect Frame(AutomationElement el) + { + try { return el.Current.BoundingRectangle; } catch (Exception) { return Rect.Empty; } + } + + /// The child window under a screen point (deepest), to post a click to. + IntPtr WindowAt(POINT screen) + { + IntPtr at = Hwnd; + for (int i = 0; i < 16; i++) + { + var client = screen; + Native.ScreenToClient(at, ref client); + // CWP_SKIPINVISIBLE | CWP_SKIPDISABLED | CWP_SKIPTRANSPARENT + IntPtr child = Native.ChildWindowFromPointEx(at, client, 0x1 | 0x2 | 0x4); + if (child == IntPtr.Zero || child == at) break; + at = child; + } + return at; + } + + /// A click posted to the window under a point: reaches the control without activating the window. + public void PostClick(POINT screen, string button, int count) + { + IntPtr target = WindowAt(screen); + var client = screen; + Native.ScreenToClient(target, ref client); + IntPtr lp = Native.MakeLParam(client.X, client.Y); + int down = Native.WM_LBUTTONDOWN, up = Native.WM_LBUTTONUP, dbl = Native.WM_LBUTTONDBLCLK, mk = 1; + if (button == "right") { down = Native.WM_RBUTTONDOWN; up = Native.WM_RBUTTONUP; dbl = 0x206; mk = 2; } + else if (button == "middle") { down = Native.WM_MBUTTONDOWN; up = Native.WM_MBUTTONUP; dbl = 0x209; mk = 0x10; } + Native.PostMessage(target, Native.WM_MOUSEMOVE, IntPtr.Zero, lp); + for (int n = 1; n <= Math.Max(1, Math.Min(count, 2)); n++) + { + Native.PostMessage(target, n == 2 ? dbl : down, (IntPtr)mk, lp); + Thread.Sleep(20); + Native.PostMessage(target, up, IntPtr.Zero, lp); + Thread.Sleep(40); + } + if (button == "right") Native.PostMessage(target, Native.WM_CONTEXTMENU, target, Native.MakeLParam(screen.X, screen.Y)); + } + + static POINT Center(Rect r) + { + return new POINT { X = (int)(r.X + r.Width / 2), Y = (int)(r.Y + r.Height / 2) }; + } + + public void Press(int id) + { + var el = Node(id); + if (Press(el)) return; + var r = Frame(el); + if (r.IsEmpty) throw new HelperError("press " + id + ": the control is gone or has no size; take a new snapshot."); + PostClick(Center(r), "left", 1); + } + + public void Menu(int id) + { + var r = Frame(Node(id)); + if (r.IsEmpty) throw new HelperError("menu " + id + ": the control is gone or has no size; take a new snapshot."); + PostClick(Center(r), "right", 1); + } + + /// Focus without bringing the window forward: a posted click on the control. + public void Focus(int id) + { + var r = Frame(Node(id)); + if (r.IsEmpty) throw new HelperError("focus " + id + ": the control is gone or has no size; take a new snapshot."); + PostClick(Center(r), "left", 1); + } + + public void SetValue(int id, string text) + { + var el = Node(id); + var value = Pattern(el, ValuePattern.Pattern) as ValuePattern; + if (value == null || value.Current.IsReadOnly) + { + throw new HelperError("set_value " + id + ": this control does not take a value directly. Focus it and use type instead."); + } + try + { + value.SetValue(text); + } + catch (Exception e) + { + throw new HelperError("set_value " + id + " failed: " + e.Message); + } + } + + /// The deepest control under a screen point, found through the tree + /// (not the screen), so a covered window answers for itself. + AutomationElement ElementAt(POINT p) + { + var at = Root(); + var point = new System.Windows.Point(p.X, p.Y); + for (int depth = 0; depth < 40; depth++) + { + AutomationElement hit = null; + double best = double.MaxValue; + foreach (var child in Children(at)) + { + var r = Cached(child, AutomationElement.BoundingRectangleProperty, Rect.Empty); + if (r.IsEmpty || !r.Contains(point)) continue; + double area = r.Width * r.Height; + if (area < best) { best = area; hit = child; } + } + if (hit == null) break; + at = hit; + } + return at == Root() ? null : at; + } + + /// A window-mode click: the control's own action where it has one, else a posted click. + public void Click(POINT p, string button, int count) + { + if (button != "left" || count >= 2) + { + PostClick(p, button, count); + return; + } + var el = ElementAt(p); + if (el != null) + { + string role = RoleOf(el); + // Fields are focused by clicking; they rarely invoke. + if (role != "Edit" && role != "Document" && role != "ComboBox" && Press(el)) return; + } + PostClick(p, button, count); + } + + public void Scroll(POINT p, string direction, int amount) + { + int pages = Math.Max(1, Math.Min(amount / 5, 5)); + var el = ElementAt(p); + var walker = TreeWalker.ControlViewWalker; + for (int i = 0; i < 8 && el != null; i++) + { + var scroll = Pattern(el, ScrollPattern.Pattern) as ScrollPattern; + if (scroll != null) + { + bool vertical = direction == "up" || direction == "down"; + if (vertical ? scroll.Current.VerticallyScrollable : scroll.Current.HorizontallyScrollable) + { + var step = direction == "up" || direction == "left" ? ScrollAmount.LargeDecrement : ScrollAmount.LargeIncrement; + for (int n = 0; n < pages; n++) + { + if (vertical) scroll.ScrollVertical(step); else scroll.ScrollHorizontal(step); + } + return; + } + } + try { el = walker.GetParent(el); } catch (Exception) { break; } + } + // No scroll pattern: a wheel message to the window under the point. + IntPtr target = WindowAt(p); + int lines = Math.Max(1, Math.Min(amount, 50)); + int delta = (direction == "up" || direction == "right" ? 120 : -120) * lines; + int msg = direction == "up" || direction == "down" ? Native.WM_MOUSEWHEEL : Native.WM_MOUSEHWHEEL; + Native.PostMessage(target, msg, (IntPtr)(delta << 16), Native.MakeLParam(p.X, p.Y)); + } + + /// Does anything in this window hold keyboard focus (so typing has somewhere to go)? + public bool HasFocusedControl() + { + uint pid; + uint thread = Native.GetWindowThreadProcessId(Hwnd, out pid); + var info = new GUITHREADINFO { cbSize = System.Runtime.InteropServices.Marshal.SizeOf(typeof(GUITHREADINFO)) }; + return Native.GetGUIThreadInfo(thread, ref info) && info.hwndFocus != IntPtr.Zero; + } + + /// The window's readable text for "Ask about this window": static text, + /// field values (never passwords), link and image names, top to bottom. + public string ReadableText(RECT window, int maxChars) + { + var top = Root(); + var parts = new List(); + int total = 0, visited = 0; + var visible = new Rect(window.Left - 2, window.Top - 2, window.Width + 4, window.Height + 4); + var clock = Stopwatch.StartNew(); + Action take = s => + { + if (s == null) return; + s = s.Trim(); + if (s.Length == 0 || (parts.Count > 0 && parts[parts.Count - 1] == s)) return; + parts.Add(s); + total += s.Length + 1; + }; + Action walk = null; + walk = (el, depth) => + { + if (total >= maxChars || depth > 60 || visited > 8000 || clock.ElapsedMilliseconds > 6000) return; + visited++; + var frame = Cached(el, AutomationElement.BoundingRectangleProperty, Rect.Empty); + if (!frame.IsEmpty && frame.Width > 0 && frame.Height > 0 && !frame.IntersectsWith(visible)) return; + if (Cached(el, AutomationElement.IsPasswordProperty, false)) return; + string role = RoleOf(el); + string name = Cached(el, AutomationElement.NameProperty, null); + var kids = Children(el); + switch (role) + { + case "Text": take(name); break; + case "Edit": case "ComboBox": take(Cached(el, ValuePattern.ValueProperty, null)); break; + case "Hyperlink": if (kids.Count == 0) take(name); break; + case "Image": if (!string.IsNullOrWhiteSpace(name)) take("[image: " + name.Trim() + "]"); break; + case "ListItem": case "TreeItem": case "DataItem": if (kids.Count == 0) take(name); break; + } + foreach (var child in kids) walk(child, depth + 1); + }; + using (cache.Activate()) + { + try { top = top.GetUpdatedCache(cache); } catch (Exception) { } + } + walk(top, 0); + int waited = 0; + while (parts.Count < 5 && waited < 4) + { + Thread.Sleep(400); + waited++; + parts.Clear(); total = 0; visited = 0; + walk(top, 0); + } + if (parts.Count == 0) return null; + string text = string.Join("\n", parts); + if (text.Length > maxChars) text = text.Substring(0, maxChars) + "\n…"; + return text; + } + } +} diff --git a/native/windows/stem-computer/Watch.cs b/native/windows/stem-computer/Watch.cs new file mode 100644 index 00000000..99b31147 --- /dev/null +++ b/native/windows/stem-computer/Watch.cs @@ -0,0 +1,100 @@ +using System; +using System.Runtime.InteropServices; +using System.Threading; + +namespace StemComputer +{ + /// The kill switch: low-level mouse and keyboard hooks that report the + /// person's first touch while a screen-mode run is driving. Events this + /// helper sent come marked injected by Windows and are ignored. Starts + /// disarmed, like the Mac's Watch.swift: looking never ends a run, and the + /// first screen-mode input action arms it. + class Watch + { + Thread thread; + uint threadId; + IntPtr mouseHook = IntPtr.Zero, keyHook = IntPtr.Zero; + // Kept as fields: a hook delegate the GC collects crashes the process on the next event. + HookProc mouseProc, keyProc; + readonly object gate = new object(); + bool fired, suppressed = true; + Action onHuman; + + public void Arm() + { + lock (gate) { suppressed = false; fired = false; } + } + + public void Disarm() + { + lock (gate) { suppressed = true; } + } + + void Saw(string kind) + { + Action handler; + lock (gate) + { + if (fired || suppressed) return; + fired = true; + handler = onHuman; + } + if (handler != null) handler(kind); + } + + public void Start(Action handler) + { + Stop(); + lock (gate) { fired = false; onHuman = handler; } + mouseProc = (code, wParam, lParam) => + { + if (code >= 0) + { + var info = (MSLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(MSLLHOOKSTRUCT)); + if ((info.flags & (Native.LLMHF_INJECTED | Native.LLMHF_LOWER_IL_INJECTED)) == 0) Saw("mouse"); + } + return Native.CallNextHookEx(mouseHook, code, wParam, lParam); + }; + keyProc = (code, wParam, lParam) => + { + if (code >= 0) + { + var info = (KBDLLHOOKSTRUCT)Marshal.PtrToStructure(lParam, typeof(KBDLLHOOKSTRUCT)); + if ((info.flags & (Native.LLKHF_INJECTED | Native.LLKHF_LOWER_IL_INJECTED)) == 0) Saw("key"); + } + return Native.CallNextHookEx(keyHook, code, wParam, lParam); + }; + var ready = new ManualResetEvent(false); + string failure = null; + thread = new Thread(() => + { + threadId = Native.GetCurrentThreadId(); + IntPtr module = Native.GetModuleHandle(null); + mouseHook = Native.SetWindowsHookEx(Native.WH_MOUSE_LL, mouseProc, module, 0); + keyHook = Native.SetWindowsHookEx(Native.WH_KEYBOARD_LL, keyProc, module, 0); + if (mouseHook == IntPtr.Zero || keyHook == IntPtr.Zero) failure = "Could not watch for your input (Windows refused the input hooks)."; + ready.Set(); + if (failure != null) return; + // Low-level hooks are called on this thread's message loop. + MSG msg; + while (Native.GetMessage(out msg, IntPtr.Zero, 0, 0) > 0) { } + Native.UnhookWindowsHookEx(mouseHook); + Native.UnhookWindowsHookEx(keyHook); + mouseHook = keyHook = IntPtr.Zero; + }); + thread.IsBackground = true; + thread.Name = "stem-computer-watch"; + thread.Start(); + ready.WaitOne(); + if (failure != null) throw new HelperError(failure); + } + + public void Stop() + { + if (thread != null && threadId != 0) Native.PostThreadMessage(threadId, Native.WM_QUIT, IntPtr.Zero, IntPtr.Zero); + thread = null; + threadId = 0; + lock (gate) { onHuman = null; } + } + } +} diff --git a/native/windows/stem-computer/Windows.cs b/native/windows/stem-computer/Windows.cs new file mode 100644 index 00000000..7dfb9587 --- /dev/null +++ b/native/windows/stem-computer/Windows.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace StemComputer +{ + /// One line of `list-windows`. + class WindowInfo + { + public IntPtr Hwnd; + public uint Pid; + public string App; + public string Exe; + public string Title; + public RECT Bounds; + public bool Minimized; + /// On another virtual desktop (the shell cloaks it). + public bool OtherDesktop; + public bool Foreground; + /// Built on Chromium (Electron apps, Chrome-family browsers). + public bool Chromium; + public long Id { get { return Hwnd.ToInt64(); } } + } + + static class Windows + { + /// The process that started this helper (Stem): its windows are never listed. + public static uint ParentPid; + + static readonly Dictionary processNames = new Dictionary(); + + /// A process's name as a person would say it (its file description, + /// "Microsoft Edge"), and its executable path. + static string[] ProcessName(uint pid) + { + string[] known; + if (processNames.TryGetValue(pid, out known)) return known; + string name = "pid " + pid, exe = ""; + try + { + using (var p = Process.GetProcessById((int)pid)) + { + name = p.ProcessName; + try + { + exe = p.MainModule.FileName; + var desc = p.MainModule.FileVersionInfo.FileDescription; + if (!string.IsNullOrWhiteSpace(desc)) name = desc.Trim(); + } + catch (Exception) + { + // An elevated or protected process: its name is all we may read. + } + } + } + catch (Exception) + { + // Gone already. + } + known = new[] { name, exe }; + processNames[pid] = known; + return known; + } + + /// Top-level windows a person would call a window: visible, unowned or + /// app-styled, not a tool window, a real size, not ours. + public static List List() + { + var fg = Native.GetForegroundWindow(); + uint ours = (uint)Process.GetCurrentProcess().Id; + var outList = new List(); + Native.EnumWindows((hwnd, _) => + { + if (!Native.IsWindowVisible(hwnd)) return true; + int ex = Native.GetWindowLong(hwnd, Native.GWL_EXSTYLE); + if ((ex & Native.WS_EX_TOOLWINDOW) != 0) return true; + uint pid; + Native.GetWindowThreadProcessId(hwnd, out pid); + if (pid == ours || pid == ParentPid) return true; + string title = Native.WindowText(hwnd); + string cls = Native.ClassName(hwnd); + // The desktop and the taskbar are the shell, not windows. + if (cls == "Progman" || cls == "WorkerW" || cls == "Shell_TrayWnd" || cls == "Shell_SecondaryTrayWnd") return true; + bool minimized = Native.IsIconic(hwnd); + var frame = Native.Frame(hwnd); + if (!minimized && (frame.Width < 50 || frame.Height < 50)) return true; + bool cloaked = Native.Cloaked(hwnd); + // A cloaked window with no title is a suspended Store app's frame, not a window. + if (cloaked && title.Length == 0) return true; + var names = ProcessName(pid); + outList.Add(new WindowInfo + { + Hwnd = hwnd, + Pid = pid, + App = names[0], + Exe = names[1], + Title = title, + Bounds = frame, + Minimized = minimized, + OtherDesktop = cloaked, + Foreground = hwnd == fg, + Chromium = cls == "Chrome_WidgetWin_1" + }); + return true; + }, IntPtr.Zero); + return outList; + } + + public static string Describe(List windows) + { + if (windows.Count == 0) return "No windows are open."; + var lines = new List { "Windows (id app \"title\" where size). select_window by window_id, or by app (+ title)." }; + // EnumWindows walks the z-order, top first: the order a person sees them in. + foreach (var w in windows.OrderBy(w => w.Foreground ? 0 : 1).ThenBy(w => w.Minimized || w.OtherDesktop ? 1 : 0)) + { + string place = w.Minimized ? "minimized" : w.OtherDesktop ? "other desktop" : "on screen (maybe covered)"; + string title = w.Title.Length == 0 ? "(untitled)" : "\"" + w.Title + "\""; + string hint = w.Minimized ? " · minimized: cannot be pictured until restored" : ""; + lines.Add(w.Id + " " + w.App + " " + title + " " + place + " " + w.Bounds.Width + "x" + w.Bounds.Height + hint + (w.Foreground ? " ← frontmost" : "")); + } + return string.Join("\n", lines); + } + + public static WindowInfo Find(long? windowId, string app, string title) + { + var all = List(); + if (windowId.HasValue) + { + var hit = all.FirstOrDefault(w => w.Id == windowId.Value); + if (hit == null) throw new HelperError("No window " + windowId.Value + " is open now. Run list_windows again."); + return hit; + } + if (string.IsNullOrEmpty(app)) throw new HelperError("select_window needs window_id, or app (and optionally title)."); + string needle = app.ToLowerInvariant(); + Func exeName = w => System.IO.Path.GetFileNameWithoutExtension(w.Exe ?? "").ToLowerInvariant(); + var candidates = all.Where(w => w.App.ToLowerInvariant() == needle || exeName(w) == needle).ToList(); + if (candidates.Count == 0) candidates = all.Where(w => w.App.ToLowerInvariant().Contains(needle) || exeName(w).Contains(needle)).ToList(); + if (candidates.Count == 0) + { + var names = all.Select(w => w.App).Distinct().OrderBy(n => n); + throw new HelperError("No running app called \"" + app + "\" has a window. Apps with windows: " + string.Join(", ", names) + "."); + } + if (!string.IsNullOrEmpty(title)) + { + string t = title.ToLowerInvariant(); + candidates = candidates.Where(w => w.Title.ToLowerInvariant().Contains(t)).ToList(); + if (candidates.Count == 0) throw new HelperError(app + " has no window whose title contains \"" + title + "\". Run list_windows to see its windows."); + } + // The app's front window: not minimized, on this desktop, then the largest. + return candidates + .OrderBy(w => w.Minimized ? 1 : 0) + .ThenBy(w => w.OtherDesktop ? 1 : 0) + .ThenByDescending(w => (long)w.Bounds.Width * w.Bounds.Height) + .First(); + } + + /// A process's windows as listed, for "the app opened a new window". + public static List AppWindows(uint pid) + { + return List().Where(w => w.Pid == pid).ToList(); + } + + public static string Brief(WindowInfo w) + { + return w.Id + " " + (w.Title.Length == 0 ? "(untitled)" : "\"" + w.Title + "\"") + " " + w.Bounds.Width + "x" + w.Bounds.Height; + } + } +} diff --git a/native/windows/stem-computer/smoke.json b/native/windows/stem-computer/smoke.json new file mode 100644 index 00000000..2dcf9f54 --- /dev/null +++ b/native/windows/stem-computer/smoke.json @@ -0,0 +1,15 @@ +[ + {"cmd": "status"}, + "notepad.exe", + {"sleep": 2000}, + {"cmd": "list-windows"}, + {"cmd": "screenshot"}, + {"cmd": "select-window", "app": "notepad"}, + {"cmd": "snapshot", "depth": 14}, + {"cmd": "type", "text": "Hello from Stem window mode"}, + {"cmd": "select-window"}, + {"cmd": "peek-window"}, + {"cmd": "watch", "on": true}, + {"cmd": "move", "x": 100, "y": 100}, + {"cmd": "click", "x": 400, "y": 300} +] diff --git a/package-lock.json b/package-lock.json index 5f25a4e5..eb737195 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,21 @@ { "name": "stem", - "version": "0.6.1", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "stem", - "version": "0.6.1", + "version": "0.7.0", "hasInstallScript": true, "dependencies": { "@earendil-works/pi-coding-agent": "1.1.0", "@huggingface/transformers": "^4.3.1", "acpx": "0.13.1", "electron-updater": "^6.8.9", + "ignore": "^7.0.12", "pdfjs-dist": "^6.1.200", + "pele": "^1.6.0", "pi-web-access": "0.18.0", "remark-gfm": "^4.0.0", "remark-mdx": "^3.1.0", @@ -41,7 +43,6 @@ "eslint-plugin-react-hooks": "^5.2.0", "lowlight": "^3.3.0", "lucide-react": "^0.468.0", - "mermaid": "^11.17.2", "patch-package": "^8.0.1", "playwright": "^1.61.0", "react": "^19.1.1", @@ -63,20 +64,6 @@ "zod": "^3.25.0 || ^4.0.0" } }, - "node_modules/@antfu/install-pkg": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-2.1.0.tgz", - "integrity": "sha512-sdg9NxU3zR4Mnawfbc/x6GB5Wf17WYud5qOuEuxXjaKpYpMkISSJEjItGebXJ2bQ4DIcly4NYH23mtkGJjvKUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "package-manager-detector": "^1.8.0", - "tinyexec": "^1.3.1" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, "node_modules/@anthropic-ai/sdk": { "version": "0.129.0", "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.129.0.tgz", @@ -766,20 +753,6 @@ "node": ">=6.9.0" } }, - "node_modules/@braintree/sanitize-url": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz", - "integrity": "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@chevrotain/types": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-11.1.2.tgz", - "integrity": "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==", - "dev": true, - "license": "Apache-2.0" - }, "node_modules/@clack/core": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/@clack/core/-/core-1.4.3.tgz", @@ -1448,6 +1421,15 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ignore": { + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.8.tgz", + "integrity": "sha512-YYNsSlXBjMk92SKnkwvB5LOVSa6OznlFUGcsvrFgNJbJCd0M1XKeFVRc8ZByeCqz32FivYNHJVooLmdqrmvp/Q==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/@earendil-works/pi-coding-agent/node_modules/lru-cache": { "version": "11.4.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.4.0.tgz", @@ -2602,25 +2584,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@iconify/types": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@iconify/types/-/types-2.0.0.tgz", - "integrity": "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@iconify/utils": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-3.1.7.tgz", - "integrity": "sha512-JZHlwdID+dy+lTgbYC8NEC4zeugqeYsc6jewvzb4c58kHauJn+X7rNwQjxz5p2qSjqaEeQoLkCIQ9v/H4PK0/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@antfu/install-pkg": "^2.0.1", - "@iconify/types": "^2.0.0", - "import-meta-resolve": "^4.2.0" - } - }, "node_modules/@img/colour": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", @@ -3287,16 +3250,6 @@ "node": ">=10" } }, - "node_modules/@mermaid-js/parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-1.2.1.tgz", - "integrity": "sha512-n12NohV3mrUyUL2o93IgG/ifeW9FTyeJn3zDxkhwa8MJ9Fxg3HQMlA3RiGmD/3UnJvheztkjjQAjA2T4LmUcpw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@chevrotain/types": "~11.1.2" - } - }, "node_modules/@mixmark-io/domino": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", @@ -4246,290 +4199,6 @@ "assertion-error": "^2.0.1" } }, - "node_modules/@types/d3": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", - "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-array": "*", - "@types/d3-axis": "*", - "@types/d3-brush": "*", - "@types/d3-chord": "*", - "@types/d3-color": "*", - "@types/d3-contour": "*", - "@types/d3-delaunay": "*", - "@types/d3-dispatch": "*", - "@types/d3-drag": "*", - "@types/d3-dsv": "*", - "@types/d3-ease": "*", - "@types/d3-fetch": "*", - "@types/d3-force": "*", - "@types/d3-format": "*", - "@types/d3-geo": "*", - "@types/d3-hierarchy": "*", - "@types/d3-interpolate": "*", - "@types/d3-path": "*", - "@types/d3-polygon": "*", - "@types/d3-quadtree": "*", - "@types/d3-random": "*", - "@types/d3-scale": "*", - "@types/d3-scale-chromatic": "*", - "@types/d3-selection": "*", - "@types/d3-shape": "*", - "@types/d3-time": "*", - "@types/d3-time-format": "*", - "@types/d3-timer": "*", - "@types/d3-transition": "*", - "@types/d3-zoom": "*" - } - }, - "node_modules/@types/d3-array": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", - "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-axis": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", - "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-brush": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", - "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-chord": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", - "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-color": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", - "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-contour": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", - "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-array": "*", - "@types/geojson": "*" - } - }, - "node_modules/@types/d3-delaunay": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", - "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-dispatch": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", - "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-drag": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", - "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-dsv": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", - "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-ease": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", - "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-fetch": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", - "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-dsv": "*" - } - }, - "node_modules/@types/d3-force": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", - "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-format": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz", - "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-geo": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.1.tgz", - "integrity": "sha512-65Emv9fQiQQqphLlRkuQ5ypPsOmWPhtBGCMv61JDPEPMvsx+gzhGf74yw1a78xFKPj6zw4AgQICJoQv0vK9M2w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/geojson": "*" - } - }, - "node_modules/@types/d3-hierarchy": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", - "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-interpolate": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", - "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-color": "*" - } - }, - "node_modules/@types/d3-path": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", - "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-polygon": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", - "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-quadtree": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", - "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-random": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.4.tgz", - "integrity": "sha512-UHYId5WTCx4L4YNel7NU00XUXXgvgpgZOvp10PuvsQENjMDXhh2RyFc0KBjO7B45ne4Ha1yVH7ii0vnzKkuzWA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-scale": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", - "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-time": "*" - } - }, - "node_modules/@types/d3-scale-chromatic": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", - "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-selection": { - "version": "3.0.12", - "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.12.tgz", - "integrity": "sha512-Qe/KWYhEiIIxGs7HrAAjMfShxKldx19SJtr5zu53f3afPsdZNz7HHtdTLXo/kqeiWNXVycI24kSnfzBYkTzpgw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-shape": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.2.0.tgz", - "integrity": "sha512-kVd74ta9eof3eJOvbNd1vGKS/XERRyQbT26Og63hIsvDO84cjD5gEOhsXf26w3FSoNlPVz84DOFcKv/oou+fMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-path": "*" - } - }, - "node_modules/@types/d3-time": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", - "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-time-format": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", - "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-timer": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", - "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/d3-transition": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", - "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-zoom": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.9.tgz", - "integrity": "sha512-0sE1406XBYJGiqD3AusTl9ZqC//2mIXix51tbom25gDCA8ri4xnSZg28CaSE8Srl6FClqABUKfsn0qghgdepMA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/d3-interpolate": "*", - "@types/d3-selection": "*" - } - }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -4571,13 +4240,6 @@ "@types/node": "*" } }, - "node_modules/@types/geojson": { - "version": "7946.0.16", - "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", - "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/hast": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", @@ -4928,17 +4590,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@upsetjs/venn.js": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@upsetjs/venn.js/-/venn.js-2.0.0.tgz", - "integrity": "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw==", - "dev": true, - "license": "MIT", - "optionalDependencies": { - "d3-selection": "^3.0.0", - "d3-transition": "^3.0.1" - } - }, "node_modules/@vitejs/plugin-react": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.2.0.tgz", @@ -6154,674 +5805,113 @@ "delayed-stream": "~1.0.0" }, "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", - "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/compare-version": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/compare-version/-/compare-version-0.1.2.tgz", - "integrity": "sha512-pJDh5/4wrEnXX/VWRZvruAGHkzKdr46z11OlTPN+VrATlWWhSKewNCJ1futCO5C7eJB3nPMFZA1LeYtcFboZ2A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/cose-base": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", - "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", - "dev": true, - "license": "MIT", - "dependencies": { - "layout-base": "^1.0.0" - } - }, - "node_modules/cross-dirname": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/cross-dirname/-/cross-dirname-0.1.0.tgz", - "integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/css-select": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", - "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^6.1.0", - "domhandler": "^5.0.2", - "domutils": "^3.0.1", - "nth-check": "^2.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-what": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", - "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/cssom": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", - "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", - "license": "MIT" - }, - "node_modules/csstype": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", - "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/cytoscape": { - "version": "3.34.3", - "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.3.tgz", - "integrity": "sha512-yfYGhRcGAntq6YBD583j4n0Eg3jIxvWmZtz/5uz9UYkeIStSlMxuUja+ec5j3iBD8nv1rwaOAYMW09tBdkSeaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10" - } - }, - "node_modules/cytoscape-cose-bilkent": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", - "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "cose-base": "^1.0.0" - }, - "peerDependencies": { - "cytoscape": "^3.2.0" - } - }, - "node_modules/cytoscape-fcose": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", - "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "cose-base": "^2.2.0" - }, - "peerDependencies": { - "cytoscape": "^3.2.0" - } - }, - "node_modules/cytoscape-fcose/node_modules/cose-base": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz", - "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==", - "dev": true, - "license": "MIT", - "dependencies": { - "layout-base": "^2.0.0" - } - }, - "node_modules/cytoscape-fcose/node_modules/layout-base": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", - "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==", - "dev": true, - "license": "MIT" - }, - "node_modules/d3": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", - "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-array": "3", - "d3-axis": "3", - "d3-brush": "3", - "d3-chord": "3", - "d3-color": "3", - "d3-contour": "4", - "d3-delaunay": "6", - "d3-dispatch": "3", - "d3-drag": "3", - "d3-dsv": "3", - "d3-ease": "3", - "d3-fetch": "3", - "d3-force": "3", - "d3-format": "3", - "d3-geo": "3", - "d3-hierarchy": "3", - "d3-interpolate": "3", - "d3-path": "3", - "d3-polygon": "3", - "d3-quadtree": "3", - "d3-random": "3", - "d3-scale": "4", - "d3-scale-chromatic": "3", - "d3-selection": "3", - "d3-shape": "3", - "d3-time": "3", - "d3-time-format": "4", - "d3-timer": "3", - "d3-transition": "3", - "d3-zoom": "3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-array": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", - "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", - "dev": true, - "license": "ISC", - "dependencies": { - "internmap": "1 - 2" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-axis": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", - "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-brush": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", - "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-dispatch": "1 - 3", - "d3-drag": "2 - 3", - "d3-interpolate": "1 - 3", - "d3-selection": "3", - "d3-transition": "3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-chord": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", - "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-path": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-color": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", - "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-contour": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", - "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-array": "^3.2.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-delaunay": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", - "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", - "dev": true, - "license": "ISC", - "dependencies": { - "delaunator": "5" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-dispatch": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", - "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-drag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", - "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-dispatch": "1 - 3", - "d3-selection": "3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-dsv": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", - "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", - "dev": true, - "license": "ISC", - "dependencies": { - "commander": "7", - "iconv-lite": "0.6", - "rw": "1" - }, - "bin": { - "csv2json": "bin/dsv2json.js", - "csv2tsv": "bin/dsv2dsv.js", - "dsv2dsv": "bin/dsv2dsv.js", - "dsv2json": "bin/dsv2json.js", - "json2csv": "bin/json2dsv.js", - "json2dsv": "bin/json2dsv.js", - "json2tsv": "bin/json2dsv.js", - "tsv2csv": "bin/dsv2dsv.js", - "tsv2json": "bin/dsv2json.js" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-dsv/node_modules/commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, - "node_modules/d3-ease": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", - "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-fetch": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", - "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-dsv": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-force": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", - "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-dispatch": "1 - 3", - "d3-quadtree": "1 - 3", - "d3-timer": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-format": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", - "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-geo": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", - "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-array": "2.5.0 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-hierarchy": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", - "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-interpolate": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", - "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-color": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-path": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", - "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-polygon": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", - "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-quadtree": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", - "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-random": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", - "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-sankey": { - "version": "0.12.3", - "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", - "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "d3-array": "1 - 2", - "d3-shape": "^1.2.0" - } - }, - "node_modules/d3-sankey/node_modules/d3-array": { - "version": "2.12.1", - "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", - "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "internmap": "^1.0.0" - } - }, - "node_modules/d3-sankey/node_modules/d3-path": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", - "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/d3-sankey/node_modules/d3-shape": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", - "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "d3-path": "1" - } - }, - "node_modules/d3-sankey/node_modules/internmap": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", - "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", - "dev": true, - "license": "ISC" - }, - "node_modules/d3-scale": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", - "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-array": "2.10.0 - 3", - "d3-format": "1 - 3", - "d3-interpolate": "1.2.0 - 3", - "d3-time": "2.1.1 - 3", - "d3-time-format": "2 - 4" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-scale-chromatic": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", - "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-color": "1 - 3", - "d3-interpolate": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-selection": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", - "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-shape": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", - "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-path": "^3.1.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-time": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", - "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", - "dev": true, - "license": "ISC", - "dependencies": { - "d3-array": "2 - 3" - }, - "engines": { - "node": ">=12" + "node": ">= 0.8" } }, - "node_modules/d3-time-format": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", - "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "node_modules/commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", "dev": true, - "license": "ISC", - "dependencies": { - "d3-time": "1 - 3" - }, + "license": "MIT", "engines": { - "node": ">=12" + "node": ">= 6" } }, - "node_modules/d3-timer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", - "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "node_modules/compare-version": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/compare-version/-/compare-version-0.1.2.tgz", + "integrity": "sha512-pJDh5/4wrEnXX/VWRZvruAGHkzKdr46z11OlTPN+VrATlWWhSKewNCJ1futCO5C7eJB3nPMFZA1LeYtcFboZ2A==", "dev": true, - "license": "ISC", + "license": "MIT", "engines": { - "node": ">=12" + "node": ">=0.10.0" } }, - "node_modules/d3-transition": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", - "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true, - "license": "ISC", + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-dirname": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/cross-dirname/-/cross-dirname-0.1.0.tgz", + "integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", "dependencies": { - "d3-color": "1 - 3", - "d3-dispatch": "1 - 3", - "d3-ease": "1 - 3", - "d3-interpolate": "1 - 3", - "d3-timer": "1 - 3" + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" }, "engines": { - "node": ">=12" - }, - "peerDependencies": { - "d3-selection": "2 - 3" + "node": ">= 8" } }, - "node_modules/d3-zoom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", - "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", - "dev": true, - "license": "ISC", + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", "dependencies": { - "d3-dispatch": "1 - 3", - "d3-drag": "2 - 3", - "d3-interpolate": "1 - 3", - "d3-selection": "2 - 3", - "d3-transition": "2 - 3" + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", "engines": { - "node": ">=12" + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" } }, - "node_modules/dagre-d3-es": { - "version": "7.0.14", - "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.14.tgz", - "integrity": "sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg==", + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "dev": true, - "license": "MIT", - "dependencies": { - "d3": "^7.9.0", - "lodash-es": "^4.17.21" - } + "license": "MIT" }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", @@ -6883,13 +5973,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/dayjs": { - "version": "1.11.23", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz", - "integrity": "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==", - "dev": true, - "license": "MIT" - }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -7000,16 +6083,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/delaunator": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.1.0.tgz", - "integrity": "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "robust-predicates": "^3.0.2" - } - }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -7691,19 +6764,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/es-toolkit": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.52.0.tgz", - "integrity": "sha512-XTNEJQh1tY1ZJVcf6ayP/2n4ZPyaHlW2FWs7xvw5ddPuhUVjLD3olQVQS7kf58JbAB48iL0uL/jerTrjtV3lDA==", - "dev": true, - "license": "MIT", - "workspaces": [ - "docs", - "benchmarks", - "tests/types", - "tests/browser-compat" - ] - }, "node_modules/es6-error": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", @@ -8150,16 +7210,6 @@ "fast-string-width": "^3.0.2" } }, - "node_modules/fastdom": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fastdom/-/fastdom-1.0.12.tgz", - "integrity": "sha512-LB+xjSTEbjHE1cWsxu+tN2Xqr1kpi+V9aADI7sVM5ZMaXyYGPHULQMzpJMYqOTULK/73pUkWVzzObFRBkPr+hg==", - "dev": true, - "license": "MIT", - "dependencies": { - "strictdom": "^1.0.1" - } - }, "node_modules/fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -8779,13 +7829,6 @@ "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", "license": "ISC" }, - "node_modules/hachure-fill": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz", - "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", - "dev": true, - "license": "MIT" - }, "node_modules/has-bigints": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", @@ -8990,23 +8033,10 @@ "node": ">= 14" } }, - "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/ignore": { - "version": "7.0.8", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.8.tgz", - "integrity": "sha512-YYNsSlXBjMk92SKnkwvB5LOVSa6OznlFUGcsvrFgNJbJCd0M1XKeFVRc8ZByeCqz32FivYNHJVooLmdqrmvp/Q==", + "version": "7.0.12", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.12.tgz", + "integrity": "sha512-/8UvqAPU9DGTI9k4mxtf49U37Isfwr8Uts96+SBHIkFxPJnHS0Ew4f00sM4Scd8V8EjM0jUNtVDdv1kPdt35lg==", "license": "MIT", "engines": { "node": ">= 4" @@ -9029,17 +8059,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/import-meta-resolve": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", - "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", - "dev": true, - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -9083,16 +8102,6 @@ "node": ">= 0.4" } }, - "node_modules/internmap": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", - "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, "node_modules/is-alphabetical": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", @@ -9787,33 +8796,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/katex": { - "version": "0.16.47", - "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", - "integrity": "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==", - "dev": true, - "funding": [ - "https://opencollective.com/katex", - "https://github.com/sponsors/katex" - ], - "license": "MIT", - "dependencies": { - "commander": "^8.3.0" - }, - "bin": { - "katex": "cli.js" - } - }, - "node_modules/katex/node_modules/commander": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", - "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -9824,12 +8806,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/khroma": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", - "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==", - "dev": true - }, "node_modules/klaw-sync": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz", @@ -9840,13 +8816,6 @@ "graceful-fs": "^4.1.11" } }, - "node_modules/layout-base": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", - "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", - "dev": true, - "license": "MIT" - }, "node_modules/lazy-val": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/lazy-val/-/lazy-val-1.0.5.tgz", @@ -9903,13 +8872,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lodash-es": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", - "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.escaperegexp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", @@ -10325,50 +9287,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/mermaid": { - "version": "11.17.2", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.17.2.tgz", - "integrity": "sha512-V6K3C8EBdEsPFZXSKMJe6ppQOENxuHARr9GvHX4hh47lAbhMRD9qf4oEK7LoaRQxULMa80/qt5gHO73aCleBBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@braintree/sanitize-url": "^7.1.2", - "@iconify/utils": "^3.0.2", - "@mermaid-js/parser": "^1.2.1", - "@types/d3": "^7.4.3", - "@upsetjs/venn.js": "^2.0.0", - "cytoscape": "^3.34.0", - "cytoscape-cose-bilkent": "^4.1.0", - "cytoscape-fcose": "^2.2.0", - "d3": "^7.9.0", - "d3-sankey": "^0.12.3", - "dagre-d3-es": "7.0.14", - "dayjs": "^1.11.21", - "dompurify": "^3.3.3", - "es-toolkit": "^1.45.1", - "fastdom": "1.0.12", - "katex": "^0.16.47", - "khroma": "^2.1.0", - "marked": "^16.3.0", - "roughjs": "^4.6.6", - "stylis": "^4.3.6", - "ts-dedent": "^2.2.0", - "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" - } - }, - "node_modules/mermaid/node_modules/marked": { - "version": "16.4.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz", - "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==", - "dev": true, - "license": "MIT", - "bin": { - "marked": "bin/marked.js" - }, - "engines": { - "node": ">= 20" - } - }, "node_modules/micromark": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", @@ -11763,13 +10681,6 @@ "node": ">= 4" } }, - "node_modules/package-manager-detector": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.8.0.tgz", - "integrity": "sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A==", - "dev": true, - "license": "MIT" - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -11860,13 +10771,6 @@ "node": ">=8" } }, - "node_modules/path-data-parser": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz", - "integrity": "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==", - "dev": true, - "license": "MIT" - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -11930,6 +10834,15 @@ "url": "https://github.com/sponsors/jet2jet" } }, + "node_modules/pele": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pele/-/pele-1.6.0.tgz", + "integrity": "sha512-VfAiVItsOGlGhvnBfphdT28VCG7mXMZxQNOwAGsCtxI2ztbUsT39MraEOYjsapLxVETkdUZscm0DdotYDQMLdw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -12102,24 +11015,6 @@ "node": ">=10.4.0" } }, - "node_modules/points-on-curve": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", - "integrity": "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==", - "dev": true, - "license": "MIT" - }, - "node_modules/points-on-path": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/points-on-path/-/points-on-path-0.2.1.tgz", - "integrity": "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-data-parser": "0.1.0", - "points-on-curve": "0.2.0" - } - }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -12644,13 +11539,6 @@ "node": ">=8.0" } }, - "node_modules/robust-predicates": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", - "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", - "dev": true, - "license": "Unlicense" - }, "node_modules/rollup": { "version": "4.62.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.0.tgz", @@ -12696,26 +11584,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/roughjs": { - "version": "4.6.6", - "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", - "integrity": "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "hachure-fill": "^0.5.2", - "path-data-parser": "^0.1.0", - "points-on-curve": "^0.2.0", - "points-on-path": "^0.2.1" - } - }, - "node_modules/rw": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", - "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/safe-array-concat": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz", @@ -12786,13 +11654,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true, - "license": "MIT" - }, "node_modules/sanitize-filename": { "version": "1.6.4", "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.4.tgz", @@ -13212,13 +12073,6 @@ "text-decoder": "^1.1.0" } }, - "node_modules/strictdom": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/strictdom/-/strictdom-1.0.1.tgz", - "integrity": "sha512-cEmp9QeXXRmjj/rVp9oyiqcvyocWab/HaoN4+bwFeZ7QzykJD6L3yD4v12K1x0tHpqRqVpJevN3gW7kyM39Bqg==", - "dev": true, - "license": "MIT" - }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -13341,13 +12195,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/stylis": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.4.0.tgz", - "integrity": "sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA==", - "dev": true, - "license": "MIT" - }, "node_modules/sumchecker": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz", @@ -13599,16 +12446,6 @@ "typescript": ">=4.8.4" } }, - "node_modules/ts-dedent": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.3.0.tgz", - "integrity": "sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.10" - } - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -14461,20 +13298,6 @@ "dev": true, "license": "MIT" }, - "node_modules/uuid": { - "version": "14.0.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", - "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", diff --git a/package.json b/package.json index 9d654b58..645f9845 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stem", - "version": "0.6.1", + "version": "0.7.0", "description": "An isolated, pi-backed personal assistant with rich MDX output, app-scoped skills, MCP, and Stem Recall memory.", "author": { "name": "Vladimir Schmotzer", @@ -17,6 +17,7 @@ "predev": "npm run preflight && node scripts/brand-electron-dev.mjs", "dev": "electron-vite dev", "build": "npm run typecheck && electron-vite build", + "build:share": "vite build --config vite.share.config.ts", "preview": "electron-vite preview", "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json", "lint": "eslint .", @@ -30,6 +31,7 @@ "dist": "npm run build && electron-builder --publish never", "dist:mac": "npm run build && electron-builder --mac --publish never", "dist:linux": "npm run build && electron-builder --linux --publish never", + "dist:win": "npm run build && electron-builder --win --publish never", "eval:retrieval": "node scripts/recall-eval.mjs", "eval:memory": "node scripts/memory-eval.mjs", "eval:skills": "node scripts/skill-author-eval.mjs", @@ -44,7 +46,9 @@ "@huggingface/transformers": "^4.3.1", "acpx": "0.13.1", "electron-updater": "^6.8.9", + "ignore": "^7.0.12", "pdfjs-dist": "^6.1.200", + "pele": "^1.6.0", "pi-web-access": "0.18.0", "remark-gfm": "^4.0.0", "remark-mdx": "^3.1.0", @@ -72,7 +76,6 @@ "eslint-plugin-react-hooks": "^5.2.0", "lowlight": "^3.3.0", "lucide-react": "^0.468.0", - "mermaid": "^11.17.2", "patch-package": "^8.0.1", "playwright": "^1.61.0", "react": "^19.1.1", diff --git a/relay/.gitignore b/relay/.gitignore new file mode 100644 index 00000000..5f3e7533 --- /dev/null +++ b/relay/.gitignore @@ -0,0 +1,3 @@ +/data/ +/relay.toml +/stem-relay diff --git a/relay/Dockerfile b/relay/Dockerfile new file mode 100644 index 00000000..1511d162 --- /dev/null +++ b/relay/Dockerfile @@ -0,0 +1,17 @@ +# Small static image for the Stem relay. +# docker build -t stem-relay relay/ +# docker run -v $PWD/relay-data:/data -p 8080:8080 stem-relay +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /stem-relay ./cmd/stem-relay + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /stem-relay /stem-relay +WORKDIR /data +VOLUME /data +EXPOSE 8080 +ENTRYPOINT ["/stem-relay"] +CMD ["serve", "-config", "/data/relay.toml"] diff --git a/relay/PROTOCOL.md b/relay/PROTOCOL.md new file mode 100644 index 00000000..63710ba0 --- /dev/null +++ b/relay/PROTOCOL.md @@ -0,0 +1,293 @@ +# Stem relay protocol, v1 + +The wire contract between Stems and relays. Design and reasoning live in +`docs/inter-stem-mail-design.md`; this file is what both implementations +(the Go relay in this directory, the TypeScript client in Stem) must match. +`testdata/vectors.json` holds byte-exact examples for every signed or sealed +structure. + +Conventions: JSON bodies, UTF-8. Binary values are standard base64 with +padding (`b64`). Hashes are lowercase hex SHA-256 (`hex`). Times are Unix +milliseconds. Addresses are lowercased and trimmed before any use; the domain +is the part after the last `@`. + +## Keys + +A Stem has two keypairs: + +- **identity**: Ed25519. Signs requests and envelopes. Its fingerprint is + `hex(sha256(raw 32-byte public key))`. +- **encryption**: X25519. Mail is sealed to it. + +The relay knows the identity public key (`authKey`, it authenticates requests +with it). The encryption key travels inside the **key blob**, which the relay +stores and returns but never parses: + +```json +{"v":1,"sig":"","enc":"","encSig":""} +``` + +`encSig` is the identity key's signature over `"stem-key-v1\n" + enc` (the +base64 text). Stems reject a blob whose `sig` differs from the `authKey` +returned beside it or whose `encSig` fails. + +## Request authentication + +Requests to a Stem's own relay (and lookups on any relay) are signed by the +identity key: + +``` +Stem-Key: +Stem-Time: +Stem-Sig: + +"stem-req-v1\n" + METHOD + "\n" + path-with-query + "\n" + Stem-Time + "\n" + hex(sha256(body)) +``` + +The relay rejects a skew over 5 minutes. An empty body hashes the empty +string. + +Requests to a relay that is **not** the requester's own (lookups, invites, +status) also carry the requester's sender certificate, proving which address +is asking: + +``` +Stem-Cert: +``` + +The relay verifies the certificate (see below) and that its `keyFp` matches +`Stem-Key`. Limits for these requests are keyed by the certified address. + +## Relay identity and authority + +Each relay has an Ed25519 signing key (or two during rotation), each with a +short id (`kid`). + +`GET /.well-known/stem-relay` (public): + +```json +{"v":1,"url":"https://relay.example","global":false, + "keys":[{"kid":"k1","pub":""}]} +``` + +**Which relay is authoritative for an address** comes from DNS. A TXT record +at `_stem.`: + +``` +v=stem1; relay=https://relay.acme.com; k=k1:; k=k2: +``` + +Several `k=` entries mean rotation; all are valid. A domain with no record +belongs to the **global relay**, whose URL and keys Stems ship with and +relays are configured with. A relay only registers, certifies and serves +addresses it is authoritative for; the global relay refuses domains that +have a record. + +`access=closed` in the record marks a **closed domain**: its people talk +only among themselves. Every relay that doesn't serve the domain refuses +certificates for its addresses (`403 closed_relay` on delivery, `404 +not_reachable` on lookups). Fields a relay doesn't know are ignored, so an +older relay reads the record as open. + +``` +v=stem1; relay=https://relay.acme.com; k=k1:; access=closed +``` + +A **closed relay** (`closed = true`) also refuses senders outside its own +domains and its `partner_domains`, the same way: their mail gets `403 +closed_relay`, their lookups `404 not_reachable`, so outsiders can't tell +who is on it. + +## Sender certificates + +```json +{"v":1,"address":"vlado@awantech.sk","keyFp":"","issuer":"https://relay.example", + "kid":"k1","iat":1760000000000,"exp":1760086400000,"sig":""} +``` + +`sig` is the relay key's signature over + +``` +"stem-cert-v1\n" + address + "\n" + keyFp + "\n" + issuer + "\n" + iat + "\n" + exp + "\n" + kid +``` + +Validity is 24 hours. A verifier checks: the issuer key `kid` is one the +address's authority publishes (DNS record, or the global relay's keys), the +signature, `iat <= now + 5 min`, `now < exp`. + +## Envelope + +```json +{"v":1,"id":"<32 hex>","ts":1760000000000,"from":"vlado@awantech.sk","to":"martin@gmail.com", + "cert":{...},"signer":"","sealed":"","sig":""} +``` + +`sig` is the signer's signature over + +``` +"stem-env-v1\n" + id + "\n" + ts + "\n" + from + "\n" + to + "\n" + hex(sha256(sealed bytes)) + "\n" + hex(sha256(certString)) +``` + +where `certString` is the certificate's signing string plus `"\n" + cert.sig`. + +Valid when: the certificate is valid for `from`, `hex(sha256(signer)) == +cert.keyFp`, the signature verifies, `ts` is within 5 minutes of now on +arrival at the relay, and `id` hasn't been seen in the last 7 days. + +### Sealing + +``` +eph = new X25519 keypair +shared = X25519(eph.priv, recipient.enc) +key = HKDF-SHA256(ikm = shared, salt = eph.pub || recipient.enc, info = "stem-seal-v1", 32 bytes) +aad = "stem-seal-v1\n" + id + "\n" + from + "\n" + to +ct = AES-256-GCM(key, nonce = 12 zero bytes, plaintext, aad) (ciphertext || 16-byte tag) +sealed = eph.pub (32 bytes) || ct +``` + +The key is single-use, so the fixed nonce is safe. (AES-GCM rather than +ChaCha20-Poly1305 because Electron's BoringSSL has no ChaCha20 in node:crypto.) The plaintext is Stem's +inner mail format; the relay never sees it. Inner payloads carry their own +version, and a Stem shows "needs a newer Stem" for one it doesn't know. + +## Endpoints + +All under the relay URL. `S` = signed request by a key registered on this +relay (own-account calls). `SC` = signed request with `Stem-Cert`, from any +relay. `K` = signed request by any key, registered or not. Errors are +`{"error":""}` with a 4xx/5xx status. + +| Method & path | Auth | Body → response | +|---|---|---| +| `GET /v1/info` | — | → `{v, features[], limits{maxMailBytes, ...}}` | +| `POST /v1/register` | S | `{address, keyBlob, pow}` → `202 {status:"code_sent"}`, always | +| `POST /v1/verify` | S | `{address, code}` → `{cert}` or `400 invalid_code` | +| `GET /v1/verify-link?t=` | — | email link; → confirmation page only | +| `POST /v1/verify-link` | — | form `t=` → completes the verification | +| `POST /v1/cert` | S | `{address}` → `{cert}` | +| `GET /v1/keys?address=` | SC | → `{authKey, keyBlob}` or `404 not_reachable` | +| `GET /v1/keystatus?address=&fp=` | — | → `{current: bool}` | +| `POST /v1/mail` | envelope | envelope → `202 {status}` | +| `GET /v1/mail/status?id=` | SC, the envelope's signer | → `{status}` | +| `GET /v1/stream` | S | SSE of `mail` events | +| `POST /v1/ack` | S | `{ids[]}` → `{acked}` | +| `POST /v1/allow` `/v1/unallow` | S | `{address}` → `{}` | +| `POST /v1/block` `/v1/unblock` | S | `{address}` → `{}` | +| `PUT /v1/account` | S | `{unlisted}` → `{unlisted}` | +| `POST /v1/invite` | SC | `{address}` → `202 {status:"ok"}`, always | +| `POST /v1/revoke` | S | `{}` → `{}`; this key stops being certified | +| `GET /v1/unsubscribe?t=` | — | email link; → confirmation page only | +| `POST /v1/unsubscribe` | — | form `t=` → adds the address to the suppression list | +| `POST /v1/push` | K | `{token, env, wake}` → `{result:"sent"\|"gone"}`; `409 token_bound`, `502 apns_failed` | +| `POST /v1/shares?ttl=&card=` | SC | blob bytes → `{id, url, expires}`; `400 bad_ttl`, `413`, `429`, `507 too_many_links`, `507 quota_full` | +| `GET /v1/shares` | SC | → `{shares:[{id, url, size, created, expires}]}` | +| `GET /v1/shares/{id}` | — | → blob bytes; `404 not_found`, `410 {error:"gone", why:"revoked"\|"taken_down"}` | +| `DELETE /v1/shares/{id}` | SC, the owner address | → `{}` | +| `POST /v1/shares/{id}/report` | — | `{reason, detail?, contact?, key?}` → `202 {}` | + +### Register and verify + +`pow` is `{"nonce":""}` such that +`sha256("stem-pow-v1\n" + address + "\n" + Stem-Key + "\n" + nonce)` starts +with `limits.powBits` zero bits. The relay always answers `code_sent`: for a +new address it emails a 6-digit code and a link; for a registered one it +emails the owner that someone tried to add a key. Verifying binds the address +to the key (replacing any earlier key for that address) and returns the first +certificate. + +### Lookups and discoverability + +`/v1/keys` answers only to a certified requester, within per-address +limits. A target +that is unregistered, unlisted (unless it has allowed the requester's certified +address), revoked, or suppressed answers identically: `404 not_reachable`. + +### Allow and block lists + +The relay can't see whom a Stem mails on other relays, so each Stem tells its +own relay: `allow` the addresses it has mailed or accepted, `block` the ones +it blocked. Allow lists drive unlisted lookups, the one-knock rule and +delivery status; blocks drop mail silently. + +### Delivery + +`POST /v1/mail` checks the envelope, then applies, in order: recipient +authority (this relay serves `to`), sender limits, block list, one knock, +queue quotas. The answer is always `202 {"status":"sent"}` unless the +recipient has allowed the sender, in which case a full queue answers +`507 queue_full`, or the sender is shut out by a closed relay or a closed +domain (`403 closed_relay`, see above). Mail to an address nobody holds is dropped. + +`/v1/stream` (SSE) first replays every held envelope for the account's +addresses, then pushes new ones as `event: mail`, `data: `, +with a comment ping every 25 s. A Stem acks after storing; acked envelopes +are deleted. Held envelopes expire after 7 days. + +`/v1/mail/status` answers `pending`, `delivered` or `expired` only to the +envelope's signer and only if the recipient allows the sender; otherwise +`sent`. + +### Invites + +`POST /v1/invite` counts as a knock toward that address. If the address is +unregistered, not suppressed, and every invite limit allows, the relay emails +an invite naming only the requester's certified address. The answer is always +`ok`. + +Email links never act on GET: mail scanners and link previews open every +link they see, so a GET only shows a page with a button that POSTs. + +### Push gateway + +Only relays with `[push]` configured (feature `push`). A Stem without the +app's APNs key sends `token` (the device's hex APNs token), `env` +(`production`, the default, or `sandbox`) and `wake`: `{kind, approvalKind?, +conversationId?, threadId?, approvalId?, taskId?, failed?}`. `kind` is +`approval`, `turn`, `task` or `mail`, and ids match `[A-Za-z0-9_.:-]{1,128}`. +Anything else is `400 bad_wake`. + +The relay writes the alert itself from fixed wording per kind and ignores +any other field, so a Stem can't choose what a lock screen says. A token +belongs to the first key that pushes to it. Another key gets `409 +token_bound` until the owner has been idle `rebind_days`, or until Apple +reports the token gone (`{result:"gone"}`; the Stem drops it). Limits per +hour, per client IP, per key, per token and in all are in `[push]`. + +### Share links + +Only relays with `[share]` enabled (feature `share`; the global relay). A +Stem uploads a snapshot it encrypted itself: + +``` +key = 32 random bytes (never sent; the link's fragment) +nonce = 12 random bytes +blob = 0x01 || nonce || AES-256-GCM(key, nonce, gzip(snapshot JSON), aad = "stem-share-v1") +``` + +The relay checks only the version byte and the size. The link is +`url + "#" + base64url(key)` (no padding). `id` is 22 base64url characters. +A share belongs to the certified address, not the key, so another device +holding the address can list and revoke it. Limits: `limits.maxShareBytes` +per blob, `[share] per_address_day`, `max_live` working links and +`quota_bytes` per address, and `maxShareTtlDays`: the longest `ttl`, and the +expiry when none is given (0: a share may never expire; the global relay +uses 30). Expired shares are deleted; revoked and taken-down ones answer +`410` for a year. + +`card` (optional) is the link preview the sharer publishes in the clear: +base64url (no padding) of `{"t": title, "d": description}`. It rides in the +signed path; the relay keeps one printable line of each (120 and 300 +characters). With `[share] viewer_dir` set, `GET /s/{id}` answers the +viewer's `index.html` with the `` block replaced by that +card's `og:` tags (a removed link gets a "removed" card, a cardless one the +page as built). Every share response carries `X-Robots-Tag: noindex`. + +`report` reasons: `illegal`, `privacy`, `harassment`, `spam`, `other`. +`key` is optional: a reporter who includes it lets the operator see the +content. + +## Features + +`/v1/info` lists optional features (`invites`, `pow`, ...). Stems detect, +never assume. New fields are optional; unknown fields are ignored on both +sides. diff --git a/relay/README.md b/relay/README.md new file mode 100644 index 00000000..08c5c0c4 --- /dev/null +++ b/relay/README.md @@ -0,0 +1,89 @@ +# Stem relay + +Carries end-to-end encrypted mail between Stems. It holds a mail only until +the recipient's Stem collects it (7 days at most), verifies who sent it, and +never sees what's inside. Design: `../docs/inter-stem-mail-design.md`. Wire +protocol: `PROTOCOL.md`. + +One static binary (or a small Docker image), one SQLite file, no database +server. + +## Run a private relay for your company + +1. Build: `go build -o stem-relay ./cmd/stem-relay` (Go 1.26), or + `docker build -t stem-relay .` +2. `./stem-relay init` writes `relay.toml`. Set: + - `url`: the public HTTPS address, e.g. `https://relay.acme.com` + - `domains`: the email domains you serve, e.g. `["acme.com"]` + - `global_url` / `global_keys`: the global relay's, so your people can + receive mail from addresses on it + - `[mail]`: SMTP settings for verification codes (`driver = "smtp"`) + - either `tls = true` + `tls_host` (built-in Let's Encrypt on :443), or + `listen = ":8080"` behind your own reverse proxy (`trust_proxy = true`) +3. `./stem-relay dns` prints the TXT record. Add it to your DNS: + `_stem.acme.com. TXT "v=stem1; relay=https://relay.acme.com; k=k1:..."` +4. `./stem-relay serve` + +Invites are off unless `[invites] enabled = true`. + +### Keep it inside the company + +Set `closed = true` in `relay.toml` and restart. Then: + +- only addresses in `domains` (and `partner_domains`, if you list any) can + mail your people or look up their keys; everyone else, the global relay's + users included, is refused and can't tell who is on the relay; +- `global_url` / `global_keys` are no longer needed; +- `stem-relay dns` adds `access=closed` to the record. Publish it: the + global relay (and every other relay) then refuses your people as senders, + so a colleague's unmanaged Stem can't mail out either. + +A partner domain must not be closed itself, or its relay refuses yours. + +The relay can't stop a laptop from talking to other relays. For that, install +the Stem policy file on company machines (`docs/stem-policy.md`): it turns +mail with other Stems off, or limits it to your domains through your relay, +and the app never contacts the global relay. Blocking +`relay.stem.awantech.sk` at the firewall backs it up. + +## Run the global relay + +Same, with `global = true` and no `domains`: it serves every domain without a +`_stem` record. Turn on invites and give them their own sender subdomain and +provider stream (`invite_from`, `invite_stream_header`), separate from +verification mail. Point the provider's bounce and complaint webhooks at +`POST /v1/hooks/mail-event` (header `X-Stem-Webhook-Secret`, body +`{"tag":"","kind":"bounce"|"complaint"}`). + +## Push gateway (global relay) + +Stems that don't hold the iOS app's APNs key wake their phones through the +relay. Put the team's `.p8` beside the relay, readable by uid 65532, and add: + +```toml +[push] +enabled = true +key_path = "/secrets/apns.p8" +key_id = "ND4436FVW9" +team_id = "AX23G9CAL9" +bundle_id = "sk.awantech.stem" +``` + +`/v1/info` then lists `push`. Protocol: `PROTOCOL.md`, "Push gateway". + +## Tuning + +Every limit is in `[limits]` and `[invites]` in `relay.toml`. Edit and +restart; no new release needed. + +## Signing key rotation + +1. `stem-relay keys add k2`, then `stem-relay dns` and publish both keys. +2. When DNS has spread: `stem-relay keys activate k2`. +3. After 24 hours (the certificate lifetime): `stem-relay keys remove k1` + and republish the record. + +## Development + +`go test ./...`. `testdata/vectors.json` is the byte-exact contract with +Stem's TypeScript client; `go test ./internal/proto -update` regenerates it. diff --git a/relay/cmd/stem-relay/main.go b/relay/cmd/stem-relay/main.go new file mode 100644 index 00000000..5c4daee5 --- /dev/null +++ b/relay/cmd/stem-relay/main.go @@ -0,0 +1,313 @@ +// Command stem-relay runs a Stem relay. See relay/README.md. +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "errors" + "flag" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "golang.org/x/crypto/acme/autocert" + + "github.com/join3r/stem/relay/internal/apns" + "github.com/join3r/stem/relay/internal/authority" + "github.com/join3r/stem/relay/internal/config" + "github.com/join3r/stem/relay/internal/mailer" + "github.com/join3r/stem/relay/internal/proto" + "github.com/join3r/stem/relay/internal/server" + "github.com/join3r/stem/relay/internal/store" +) + +const usage = `stem-relay — a Stem relay + + stem-relay init [-config relay.toml] write a starter config + stem-relay serve [-config relay.toml] run the relay + stem-relay dns [-config relay.toml] print the _stem DNS record (private relays) + stem-relay keys list|add|activate|remove [kid] manage signing keys + stem-relay suspend
stop an address sending, looking up and sharing + stem-relay unsuspend
lift a suspension + stem-relay share-reports list open reports on share links + stem-relay share-takedown remove a share link and close its reports + stem-relay version +` + +func main() { + log.SetFlags(log.LstdFlags) + if len(os.Args) < 2 { + fmt.Print(usage) + os.Exit(2) + } + cmd, args := os.Args[1], os.Args[2:] + fs := flag.NewFlagSet(cmd, flag.ExitOnError) + cfgPath := fs.String("config", "relay.toml", "config file") + _ = fs.Parse(args) + var err error + switch cmd { + case "init": + err = initConfig(*cfgPath) + case "serve": + err = serve(*cfgPath) + case "dns": + err = withServer(*cfgPath, printDNS) + case "keys": + err = keys(*cfgPath, fs.Args()) + case "unsuspend": + err = withStore(*cfgPath, func(_ *config.Config, st *store.Store) error { + if fs.NArg() != 1 { + return errors.New("usage: stem-relay unsuspend
") + } + return st.Unsuspend(proto.NormalizeAddress(fs.Arg(0))) + }) + case "suspend": + err = withStore(*cfgPath, func(_ *config.Config, st *store.Store) error { + if fs.NArg() != 1 { + return errors.New("usage: stem-relay suspend
") + } + return st.Suspend(proto.NormalizeAddress(fs.Arg(0))) + }) + case "share-reports": + err = withStore(*cfgPath, shareReports) + case "share-takedown": + err = withStore(*cfgPath, func(_ *config.Config, st *store.Store) error { + if fs.NArg() != 2 { + return errors.New("usage: stem-relay share-takedown ") + } + id := fs.Arg(0) + sh, err := st.GetShare(id) + if err != nil { + return fmt.Errorf("share %s: %w", id, err) + } + if _, err := st.RemoveShare(id, "takedown: "+fs.Arg(1)); err != nil { + return err + } + if err := st.HandleShareReports(id); err != nil { + return err + } + fmt.Printf("removed %s (shared by %s); suspend them with: stem-relay suspend %s\n", id, sh.Owner, sh.Owner) + return nil + }) + case "version": + fmt.Println(server.Version) + default: + fmt.Print(usage) + os.Exit(2) + } + if err != nil { + log.Fatal(err) + } +} + +func initConfig(path string) error { + c := config.Defaults() + c.URL = "https://relay.example.com" + c.Domains = []string{"example.com"} + c.GlobalURL = "https://relay.stem.example" + c.GlobalKeys = []string{"k1:"} + c.Mail.VerifyFrom = "Stem " + c.Mail.InviteFrom = "Stem " + if err := c.Write(path); err != nil { + return err + } + fmt.Printf("Wrote %s. Edit url, domains (or global = true), mail settings, then:\n stem-relay serve -config %s\n", path, path) + return nil +} + +func withStore(path string, fn func(*config.Config, *store.Store) error) error { + cfg, err := config.Load(path) + if err != nil { + return err + } + if err := os.MkdirAll(cfg.DataDir, 0o700); err != nil { + return err + } + st, err := store.Open(filepath.Join(cfg.DataDir, "relay.db")) + if err != nil { + return err + } + defer st.Close() + return fn(cfg, st) +} + +func newServer(cfg *config.Config, st *store.Store) (*server.Server, error) { + gk := map[string]ed25519.PublicKey{} + if !cfg.Global { + var err error + if gk, err = config.ParseKeys(cfg.GlobalKeys); err != nil { + return nil, err + } + } + res := authority.NewResolver(cfg.GlobalURL, gk) + return server.New(cfg, st, mailer.New(cfg.Mail), res) +} + +func withServer(path string, fn func(*config.Config, *server.Server) error) error { + return withStore(path, func(cfg *config.Config, st *store.Store) error { + s, err := newServer(cfg, st) + if err != nil { + return err + } + return fn(cfg, s) + }) +} + +func printDNS(cfg *config.Config, s *server.Server) error { + if cfg.Global { + fmt.Println("The global relay needs no DNS record. Stems ship with its URL and these keys:") + } + pubs := s.PublicKeys() + kids := make([]string, 0, len(pubs)) + for k := range pubs { + kids = append(kids, k) + } + sort.Strings(kids) + parts := []string{"v=stem1", "relay=" + cfg.URL} + for _, k := range kids { + parts = append(parts, "k="+k+":"+proto.B64(pubs[k])) + } + if cfg.Closed { + parts = append(parts, "access=closed") + } + rec := strings.Join(parts, "; ") + for _, d := range cfg.Domains { + fmt.Printf("_stem.%s. TXT \"%s\"\n", d, rec) + } + if cfg.Global { + fmt.Println(rec) + } + return nil +} + +func keys(path string, args []string) error { + return withStore(path, func(cfg *config.Config, st *store.Store) error { + if len(args) == 0 { + args = []string{"list"} + } + switch args[0] { + case "list": + ks, err := st.RelayKeys() + if err != nil { + return err + } + for _, k := range ks { + mark := "" + if k.Active { + mark = " (active)" + } + fmt.Printf("%s %s%s\n", k.Kid, proto.B64(ed25519.PrivateKey(k.Priv).Public().(ed25519.PublicKey)), mark) + } + return nil + case "add": + if len(args) != 2 { + return errors.New("usage: keys add ") + } + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + if err := st.AddRelayKey(args[1], priv, false); err != nil { + return err + } + fmt.Println("Added (inactive). Publish it (stem-relay dns), wait for DNS to spread, then: keys activate", args[1]) + return nil + case "activate": + if len(args) != 2 { + return errors.New("usage: keys activate ") + } + return st.ActivateRelayKey(args[1]) + case "remove": + if len(args) != 2 { + return errors.New("usage: keys remove ") + } + fmt.Println("Removed. Certificates it signed stop verifying; wait at least 24 h after activating its successor.") + return st.RemoveRelayKey(args[1]) + } + return errors.New("usage: keys list|add|activate|remove [kid]") + }) +} + +func serve(path string) error { + return withServer(path, func(cfg *config.Config, s *server.Server) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if p := cfg.Push; p.Enabled { + c, err := apns.New(p.KeyPath, p.KeyID, p.TeamID, p.BundleID) + if err != nil { + return err + } + s.Pusher = c + log.Printf("push gateway on for %s", p.BundleID) + } + go s.Run(ctx) + + // ReadTimeout bounds slow uploads; no WriteTimeout because /v1/stream + // is long-lived. + srv := &http.Server{Handler: s.Handler(), ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute, MaxHeaderBytes: 64 << 10} + errc := make(chan error, 1) + if cfg.TLS { + m := &autocert.Manager{ + Prompt: autocert.AcceptTOS, + HostPolicy: autocert.HostWhitelist(cfg.TLSHost), + Cache: autocert.DirCache(filepath.Join(cfg.DataDir, "certs")), + } + srv.Addr = ":443" + srv.TLSConfig = &tls.Config{GetCertificate: m.GetCertificate, MinVersion: tls.VersionTLS12} + go func() { errc <- http.ListenAndServe(":80", m.HTTPHandler(nil)) }() + go func() { errc <- srv.ListenAndServeTLS("", "") }() + log.Printf("stem-relay %s serving %s on :443", server.Version, cfg.URL) + } else { + srv.Addr = cfg.Listen + go func() { errc <- srv.ListenAndServe() }() + log.Printf("stem-relay %s serving %s on %s", server.Version, cfg.URL, cfg.Listen) + } + select { + case err := <-errc: + return err + case <-ctx.Done(): + sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return srv.Shutdown(sctx) + } + }) +} + +func shareReports(cfg *config.Config, st *store.Store) error { + reps, err := st.OpenShareReports() + if err != nil { + return err + } + if len(reps) == 0 { + fmt.Println("no open reports") + return nil + } + for _, r := range reps { + owner := "(gone)" + if sh, err := st.GetShare(r.Share); err == nil { + owner = sh.Owner + } + link := cfg.Share.ViewerURL + "/s/" + r.Share + if r.Key != "" { + link += "#" + r.Key + } + fmt.Printf("#%d %s %s by %s\n %s\n", r.ID, time.UnixMilli(r.At).UTC().Format(time.RFC3339), r.Reason, owner, link) + if r.Detail != "" { + fmt.Printf(" %s\n", strings.ReplaceAll(r.Detail, "\n", "\n ")) + } + if r.Contact != "" { + fmt.Printf(" contact: %s\n", r.Contact) + } + } + return nil +} diff --git a/relay/deploy/global/Caddyfile.snippet b/relay/deploy/global/Caddyfile.snippet new file mode 100644 index 00000000..5be81c82 --- /dev/null +++ b/relay/deploy/global/Caddyfile.snippet @@ -0,0 +1,77 @@ + +# Stem global relay (~/services/stem-relay, container stem-relay). +relay.stem.awantech.sk { + import security_headers + import cloudflare_dns + # GET /v1/stream is SSE — never buffer it. + reverse_proxy stem-relay:8080 { + flush_interval -1 + stream_close_delay 5m + } + log { + output stdout + format console + } +} + +# heystem.app zone: separate Cloudflare token (Zone:Read + DNS:Edit on heystem.app, +# IP-filtered to this box), CLOUDFLARE_API_TOKEN_HEYSTEM in ~/services/caddy/.env +# and passed through docker-compose.yml. +(cloudflare_dns_heystem) { + tls { + dns cloudflare {env.CLOUDFLARE_API_TOKEN_HEYSTEM} + } +} + +# Stem global relay on the product domain (same container as relay.stem.awantech.sk). +relay.heystem.app { + import security_headers + import cloudflare_dns_heystem + # GET /v1/stream is SSE — never buffer it. + reverse_proxy stem-relay:8080 { + flush_interval -1 + stream_close_delay 5m + } +} + +# Share-link viewer. stem-relay serves it all from ~/services/stem-relay/viewer +# (npm run build:share → dist/share, mounted at /viewer): /s/ with the +# share's link-preview tags filled in, the hashed assets, and /v1/shares/*. +# Nothing else of the relay API is reachable here. Its own origin, so shared +# content never runs where relay calls or a future heystem.app login do. The +# decryption key is in the URL fragment, which never reaches a server. +# noindex everywhere, but no robots.txt Disallow: that would stop the +# chat-app crawlers that draw link previews too. +share.heystem.app { + import cloudflare_dns_heystem + header { + Content-Security-Policy "default-src 'none'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:; connect-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" + Referrer-Policy "no-referrer" + X-Content-Type-Options "nosniff" + X-Robots-Tag "noindex, nofollow, noarchive" + Strict-Transport-Security "max-age=31536000" + Permissions-Policy "camera=(), microphone=(), geolocation=()" + -Server + } + @viewer path /s/* /assets/* /og.png /favicon.png /v1/shares /v1/shares/* + handle @viewer { + reverse_proxy stem-relay:8080 + } + handle / { + redir https://heystem.app 302 + } + handle { + respond "Not found" 404 + } +} + +# Homepage placeholder until the site exists. +heystem.app { + import cloudflare_dns_heystem + redir https://github.com/join3r/stem 302 +} + +www.heystem.app { + import cloudflare_dns_heystem + redir https://heystem.app{uri} 301 +} diff --git a/relay/deploy/global/docker-compose.yml b/relay/deploy/global/docker-compose.yml new file mode 100644 index 00000000..27a09e30 --- /dev/null +++ b/relay/deploy/global/docker-compose.yml @@ -0,0 +1,25 @@ +# Stem global relay (relay.heystem.app, formerly relay.stem.awantech.sk), deployed at root@167.233.132.66 +# in /root/services/stem-relay. Source in ./src (git archive of +# stem/desktop relay/). Caddy (/root/services/caddy) proxies to stem-relay:8080 +# on the external `edge` network; no published port. +# docker compose build && docker compose up -d +services: + stem-relay: + build: ./src + image: stem-relay:local + container_name: stem-relay + restart: unless-stopped + volumes: + - ./data:/data + # The Awantech team .p8 for the push gateway, owned by 65532, 0400. + - ./secrets/apns.p8:/secrets/apns.p8:ro + # The share viewer build (npm run build:share → dist/share), served by + # the relay on share.heystem.app (previews filled into /s/). + - ./viewer:/viewer:ro + networks: + - edge + +networks: + edge: + name: edge + external: true diff --git a/relay/deploy/global/relay.toml b/relay/deploy/global/relay.toml new file mode 100644 index 00000000..a00a1914 --- /dev/null +++ b/relay/deploy/global/relay.toml @@ -0,0 +1,40 @@ +url = "https://relay.heystem.app" +global = true +data_dir = "/data" +listen = ":8080" +trust_proxy = true + +[mail] +# Verification codes go out through Fastmail (heystem.app is a Fastmail +# domain: MX, SPF, DKIM fm1-3 and DMARC on Cloudflare). smtp_pass is a +# Fastmail app password with SMTP access; it lives only in the deployed +# data/relay.toml. +driver = "smtp" +smtp_host = "smtp.fastmail.com" +smtp_port = 587 +smtp_user = "FASTMAIL_LOGIN" +smtp_pass = "FASTMAIL_APP_PASSWORD" +verify_from = "Stem " +invite_from = "Stem " + +[invites] +enabled = false + +# APNs gateway for every Stem without the app's key (relay/README.md). +[push] +enabled = true +key_path = "/secrets/apns.p8" +key_id = "ND4436FVW9" +team_id = "AX23G9CAL9" +bundle_id = "sk.awantech.stem" + +# Public share links: encrypted blobs, viewer at share.heystem.app (Caddy +# serves the static viewer and proxies /v1/shares/* here). +[share] +enabled = true +viewer_url = "https://share.heystem.app" +viewer_dir = "/viewer" +# Abuse limits: links last at most 30 days, 100 working links per address. +max_ttl_days = 30 +max_live = 100 +per_address_day = 30 diff --git a/relay/go.mod b/relay/go.mod new file mode 100644 index 00000000..3805821d --- /dev/null +++ b/relay/go.mod @@ -0,0 +1,23 @@ +module github.com/join3r/stem/relay + +go 1.26.0 + +require ( + github.com/BurntSushi/toml v1.6.0 + golang.org/x/crypto v0.58.0 + modernc.org/sqlite v1.60.1 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + modernc.org/libc v1.77.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/relay/go.sum b/relay/go.sum new file mode 100644 index 00000000..96067374 --- /dev/null +++ b/relay/go.sum @@ -0,0 +1,58 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= +modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= +modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= +modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= +modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/relay/internal/apns/apns.go b/relay/internal/apns/apns.go new file mode 100644 index 00000000..5cb69d25 --- /dev/null +++ b/relay/internal/apns/apns.go @@ -0,0 +1,155 @@ +// Package apns sends alert pushes to Apple with a provider token (.p8). It is +// the relay's half of Stem's push gateway: Stems without Awantech's APNs key +// hand a device token and a wake-up to the relay, which signs and sends it. +package apns + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "os" + "sync" + "time" +) + +type Result int + +const ( + Sent Result = iota + // Gone: Apple says the token is dead (app deleted, device restored). + Gone + Failed +) + +var Hosts = map[string]string{ + "production": "https://api.push.apple.com", + "sandbox": "https://api.sandbox.push.apple.com", +} + +// Apple refuses a provider token older than an hour and rate-limits minting. +const tokenTTL = 50 * time.Minute + +type Client struct { + KeyID, TeamID, Topic string + key *ecdsa.PrivateKey + HTTP *http.Client + // Hosts by environment; tests point both at an httptest server. + Hosts map[string]string + + mu sync.Mutex + token string + mintedAt time.Time +} + +func New(keyPath, keyID, teamID, topic string) (*Client, error) { + raw, err := os.ReadFile(keyPath) + if err != nil { + return nil, err + } + key, err := ParseKey(raw) + if err != nil { + return nil, err + } + return &Client{KeyID: keyID, TeamID: teamID, Topic: topic, key: key, Hosts: Hosts, + HTTP: &http.Client{Timeout: 10 * time.Second}}, nil +} + +// ParseKey reads an Apple .p8 (PKCS#8 PEM, P-256). +func ParseKey(raw []byte) (*ecdsa.PrivateKey, error) { + block, _ := pem.Decode(raw) + if block == nil { + return nil, errors.New("apns key: no PEM block") + } + k, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("apns key: %w", err) + } + ec, ok := k.(*ecdsa.PrivateKey) + if !ok { + return nil, errors.New("apns key: not an EC key") + } + return ec, nil +} + +// WithKey builds a client around an already parsed key (tests). +func WithKey(key *ecdsa.PrivateKey, keyID, teamID, topic string, hc *http.Client, hosts map[string]string) *Client { + return &Client{KeyID: keyID, TeamID: teamID, Topic: topic, key: key, HTTP: hc, Hosts: hosts} +} + +func b64url(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } + +// providerToken is {alg:ES256,kid} over {iss,iat}, signed as raw r||s. +func (c *Client) providerToken() (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.token != "" && time.Since(c.mintedAt) < tokenTTL { + return c.token, nil + } + h, _ := json.Marshal(map[string]string{"alg": "ES256", "kid": c.KeyID}) + cl, _ := json.Marshal(map[string]any{"iss": c.TeamID, "iat": time.Now().Unix()}) + in := b64url(h) + "." + b64url(cl) + sum := sha256.Sum256([]byte(in)) + r, s, err := ecdsa.Sign(rand.Reader, c.key, sum[:]) + if err != nil { + return "", err + } + sig := make([]byte, 64) + r.FillBytes(sig[:32]) + s.FillBytes(sig[32:]) + c.token, c.mintedAt = in+"."+b64url(sig), time.Now() + return c.token, nil +} + +func (c *Client) forgetToken() { + c.mu.Lock() + c.token = "" + c.mu.Unlock() +} + +// Send posts one alert push. env is "production" or "sandbox". +func (c *Client) Send(ctx context.Context, env, deviceToken string, payload []byte) (Result, string) { + host, ok := c.Hosts[env] + if !ok { + return Failed, "bad env" + } + tok, err := c.providerToken() + if err != nil { + return Failed, err.Error() + } + req, err := http.NewRequestWithContext(ctx, "POST", host+"/3/device/"+deviceToken, bytes.NewReader(payload)) + if err != nil { + return Failed, err.Error() + } + req.Header.Set("authorization", "bearer "+tok) + req.Header.Set("apns-topic", c.Topic) + req.Header.Set("apns-push-type", "alert") + req.Header.Set("apns-priority", "10") + req.Header.Set("content-type", "application/json") + res, err := c.HTTP.Do(req) + if err != nil { + return Failed, err.Error() + } + defer res.Body.Close() + body, _ := io.ReadAll(io.LimitReader(res.Body, 1024)) + switch { + case res.StatusCode == 200: + return Sent, "" + case res.StatusCode == 410: + return Gone, string(body) + case res.StatusCode == 400 && bytes.Contains(body, []byte("BadDeviceToken")): + return Gone, string(body) + case res.StatusCode == 403: + c.forgetToken() + } + return Failed, fmt.Sprintf("%d %s", res.StatusCode, body) +} diff --git a/relay/internal/apns/apns_test.go b/relay/internal/apns/apns_test.go new file mode 100644 index 00000000..0facad84 --- /dev/null +++ b/relay/internal/apns/apns_test.go @@ -0,0 +1,48 @@ +package apns + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "math/big" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestSendSignsAndClassifies(t *testing.T) { + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + var gotAuth, gotTopic, gotPath string + status := 200 + hs := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth, gotTopic, gotPath = r.Header.Get("authorization"), r.Header.Get("apns-topic"), r.URL.Path + w.WriteHeader(status) + })) + defer hs.Close() + c := WithKey(key, "KID", "TEAM", "sk.awantech.stem", hs.Client(), map[string]string{"production": hs.URL}) + + if r, _ := c.Send(context.Background(), "production", "abcd", []byte(`{}`)); r != Sent { + t.Fatalf("result %v", r) + } + if gotTopic != "sk.awantech.stem" || gotPath != "/3/device/abcd" { + t.Fatalf("topic %q path %q", gotTopic, gotPath) + } + jwt := strings.TrimPrefix(gotAuth, "bearer ") + parts := strings.Split(jwt, ".") + sig, _ := base64.RawURLEncoding.DecodeString(parts[2]) + sum := sha256.Sum256([]byte(parts[0] + "." + parts[1])) + if len(sig) != 64 || !ecdsa.Verify(&key.PublicKey, sum[:], new(big.Int).SetBytes(sig[:32]), new(big.Int).SetBytes(sig[32:])) { + t.Fatal("provider token signature does not verify as raw r||s") + } + status = 410 + if r, _ := c.Send(context.Background(), "production", "abcd", []byte(`{}`)); r != Gone { + t.Fatalf("410 gave %v", r) + } + if r, _ := c.Send(context.Background(), "sandbox", "abcd", []byte(`{}`)); r != Failed { + t.Fatal("unknown env should fail") + } +} diff --git a/relay/internal/authority/authority.go b/relay/internal/authority/authority.go new file mode 100644 index 00000000..9d53e78d --- /dev/null +++ b/relay/internal/authority/authority.go @@ -0,0 +1,151 @@ +// Package authority decides which relay may certify an address: the relay +// named in the domain's `_stem.` TXT record, or the global relay. +package authority + +import ( + "context" + "crypto/ed25519" + "errors" + "net" + "strings" + "sync" + "time" + + "github.com/join3r/stem/relay/internal/proto" +) + +// Record is a parsed `_stem` TXT record. +type Record struct { + Relay string + Keys map[string]ed25519.PublicKey + // Closed (`access=closed`): the domain's people talk only among + // themselves. Every other relay refuses them as senders. + Closed bool +} + +// ParseRecord parses `v=stem1; relay=https://...; k=kid:; [access=closed]`. +// Unknown fields are ignored, so older relays read a closed record as open. +func ParseRecord(txt string) (*Record, error) { + r := &Record{Keys: map[string]ed25519.PublicKey{}} + seenV := false + for _, part := range strings.Split(txt, ";") { + k, v, ok := strings.Cut(strings.TrimSpace(part), "=") + if !ok { + continue + } + switch strings.TrimSpace(k) { + case "v": + if strings.TrimSpace(v) != "stem1" { + return nil, errors.New("unknown record version") + } + seenV = true + case "relay": + r.Relay = strings.TrimRight(strings.TrimSpace(v), "/") + case "access": + r.Closed = strings.TrimSpace(v) == "closed" + case "k": + kid, pub, ok := strings.Cut(strings.TrimSpace(v), ":") + if !ok { + continue + } + key, err := proto.ParseEd25519(pub) + if err != nil { + continue + } + r.Keys[kid] = key + } + } + if !seenV || r.Relay == "" || !strings.HasPrefix(r.Relay, "https://") && !strings.HasPrefix(r.Relay, "http://") || len(r.Keys) == 0 { + return nil, errors.New("incomplete _stem record") + } + return r, nil +} + +// TXTLookup is net.Resolver.LookupTXT's shape, swappable in tests. +type TXTLookup func(ctx context.Context, name string) ([]string, error) + +// Resolver maps addresses to their authority, caching DNS answers. +type Resolver struct { + Lookup TXTLookup + GlobalURL string + GlobalKeys map[string]ed25519.PublicKey + // Static overrides domain → record, for private setups and tests. + Static map[string]*Record + TTL time.Duration + + mu sync.Mutex + cache map[string]cached +} + +const maxCache = 10000 + +type cached struct { + rec *Record // nil = no record (global) + at time.Time +} + +func NewResolver(globalURL string, globalKeys map[string]ed25519.PublicKey) *Resolver { + return &Resolver{ + Lookup: net.DefaultResolver.LookupTXT, + GlobalURL: strings.TrimRight(globalURL, "/"), + GlobalKeys: globalKeys, + TTL: 10 * time.Minute, + } +} + +// DomainRecord returns the domain's record, or nil when the domain has none. +func (r *Resolver) DomainRecord(domain string) (*Record, error) { + if rec, ok := r.Static[domain]; ok { + return rec, nil + } + r.mu.Lock() + if c, ok := r.cache[domain]; ok && time.Since(c.at) < r.TTL { + r.mu.Unlock() + return c.rec, nil + } + r.mu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + txts, err := r.Lookup(ctx, "_stem."+domain) + var rec *Record + if err != nil { + var dnsErr *net.DNSError + if !errors.As(err, &dnsErr) || !dnsErr.IsNotFound { + return nil, err // transient: don't cache, don't guess + } + } + for _, t := range txts { + if p, perr := ParseRecord(t); perr == nil { + rec = p + break + } + } + r.mu.Lock() + // Unauthenticated mail names arbitrary sender domains; keep the cache + // from growing without bound. + if r.cache == nil || len(r.cache) >= maxCache { + r.cache = map[string]cached{} + } + r.cache[domain] = cached{rec: rec, at: time.Now()} + r.mu.Unlock() + return rec, nil +} + +// Authority returns the relay URL and keys allowed to certify the address. +func (r *Resolver) Authority(address string) (string, map[string]ed25519.PublicKey, error) { + rec, err := r.DomainRecord(proto.Domain(address)) + if err != nil { + return "", nil, err + } + if rec != nil { + return rec.Relay, rec.Keys, nil + } + if r.GlobalURL == "" { + return "", nil, errors.New("no global relay configured") + } + return r.GlobalURL, r.GlobalKeys, nil +} + +// IssuerKeys adapts Authority to proto.IssuerKeys. +func (r *Resolver) IssuerKeys() proto.IssuerKeys { return r.Authority } diff --git a/relay/internal/authority/authority_test.go b/relay/internal/authority/authority_test.go new file mode 100644 index 00000000..68d2ba73 --- /dev/null +++ b/relay/internal/authority/authority_test.go @@ -0,0 +1,59 @@ +package authority + +import ( + "context" + "crypto/ed25519" + "net" + "testing" + + "github.com/join3r/stem/relay/internal/proto" +) + +func TestParseRecord(t *testing.T) { + pub, _, _ := ed25519.GenerateKey(nil) + r, err := ParseRecord("v=stem1; relay=https://relay.acme.com/; k=k1:" + proto.B64(pub) + "; k=bad:zz") + if err != nil || r.Relay != "https://relay.acme.com" || len(r.Keys) != 1 { + t.Fatalf("%+v %v", r, err) + } + if r.Closed { + t.Fatal("a record without access=closed is open") + } + if r, err := ParseRecord("v=stem1; relay=https://relay.acme.com; k=k1:" + proto.B64(pub) + "; access=closed"); err != nil || !r.Closed { + t.Fatalf("access=closed not read: %+v %v", r, err) + } + for _, bad := range []string{"v=stem2; relay=https://x; k=k1:" + proto.B64(pub), "v=stem1; k=k1:" + proto.B64(pub), "v=stem1; relay=https://x"} { + if _, err := ParseRecord(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} + +func TestAuthority(t *testing.T) { + gpub, _, _ := ed25519.GenerateKey(nil) + apub, _, _ := ed25519.GenerateKey(nil) + res := NewResolver("https://global.test", map[string]ed25519.PublicKey{"k1": gpub}) + calls := 0 + res.Lookup = func(_ context.Context, name string) ([]string, error) { + calls++ + switch name { + case "_stem.acme.test": + return []string{"unrelated", "v=stem1; relay=https://relay.acme.test; k=a1:" + proto.B64(apub)}, nil + case "_stem.flaky.test": + return nil, &net.DNSError{Err: "timeout", IsTimeout: true} + } + return nil, &net.DNSError{Err: "no such host", IsNotFound: true} + } + if u, ks, err := res.Authority("m@acme.test"); err != nil || u != "https://relay.acme.test" || ks["a1"] == nil { + t.Fatalf("acme: %s %v", u, err) + } + if u, ks, err := res.Authority("v@gmail.test"); err != nil || u != "https://global.test" || ks["k1"] == nil { + t.Fatalf("gmail: %s %v", u, err) + } + res.Authority("m@acme.test") + if calls != 2 { + t.Fatalf("cache: %d lookups", calls) + } + if _, _, err := res.Authority("x@flaky.test"); err == nil { + t.Fatal("a DNS timeout must not fall back to the global relay") + } +} diff --git a/relay/internal/config/config.go b/relay/internal/config/config.go new file mode 100644 index 00000000..141242c8 --- /dev/null +++ b/relay/internal/config/config.go @@ -0,0 +1,244 @@ +// Package config is the relay's TOML config. Every abuse number lives +// here, so tuning a relay is a config edit, never a release. +package config + +import ( + "crypto/ed25519" + "errors" + "fmt" + "os" + "strings" + + "github.com/BurntSushi/toml" + + "github.com/join3r/stem/relay/internal/proto" +) + +type Config struct { + // Public base URL of this relay, e.g. https://relay.stem.example. + URL string `toml:"url"` + // Global is true only for the global relay: it serves every domain + // without a _stem record. A private relay sets Domains instead. + Global bool `toml:"global"` + Domains []string `toml:"domains"` + // The global relay's URL and keys ("kid:"), so a private relay can + // check certificates for addresses on the global relay. + GlobalURL string `toml:"global_url"` + GlobalKeys []string `toml:"global_keys"` + // Closed (private relays only): only addresses in Domains and + // PartnerDomains may mail this relay's people or look up their keys. + // Everyone else, the global relay's users included, is turned away. + // Publish the record with access=closed too (stem-relay dns does), so + // other relays refuse this relay's people as senders. + Closed bool `toml:"closed"` + PartnerDomains []string `toml:"partner_domains"` + + DataDir string `toml:"data_dir"` + + Listen string `toml:"listen"` // plain HTTP, e.g. ":8080" behind a proxy + TLS bool `toml:"tls"` // built-in Let's Encrypt on :443 + TLSHost string `toml:"tls_host"` // hostname for the certificate + // Trust X-Forwarded-For (only behind your own reverse proxy). + TrustProxy bool `toml:"trust_proxy"` + + Mail Mail `toml:"mail"` + Limits Limits `toml:"limits"` + Invite Invite `toml:"invites"` + Push Push `toml:"push"` + Share Share `toml:"share"` +} + +// Share hosts public share links: blobs a Stem encrypted before upload, so +// the relay never sees what is in them. The global relay only. +type Share struct { + Enabled bool `toml:"enabled"` + // Public base of the viewer, e.g. https://share.heystem.app; a link is + // /s/#. + ViewerURL string `toml:"viewer_url"` + // The viewer build (dist/share). Its index.html is served for /s/ + // with that share's link-preview tags filled in; empty = no previews. + ViewerDir string `toml:"viewer_dir"` + // Largest blob, in bytes. + MaxBytes int64 `toml:"max_bytes"` + // Shares per certified address per day, live shares it may hold at once, + // and bytes it may keep stored. + PerAddressDay int `toml:"per_address_day"` + MaxLive int `toml:"max_live"` + QuotaBytes int64 `toml:"quota_bytes"` + // Longest expiry a Stem may ask for, in days (also the default). 0 lets a + // share never expire; the global relay keeps it short so it is no free + // permanent host. + MaxTTLDays int `toml:"max_ttl_days"` + // Abuse reports per client IP per hour. + ReportsPerIPHour int `toml:"reports_per_ip_hour"` +} + +// Push is the APNs gateway: Stems without the app's APNs key send wake-ups +// here, and the relay signs them with its .p8. +type Push struct { + Enabled bool `toml:"enabled"` + KeyPath string `toml:"key_path"` + KeyID string `toml:"key_id"` + TeamID string `toml:"team_id"` + BundleID string `toml:"bundle_id"` + // Wake-ups per client IP, per Stem key, per device token, and in all, per hour. + PerIPHour int `toml:"per_ip_hour"` + PerKeyHour int `toml:"per_key_hour"` + PerTokenHour int `toml:"per_token_hour"` + GlobalHour int `toml:"global_hour"` + // A token belongs to the first Stem key that pushes to it; another key + // may take it over only after the owner has been silent this long. + RebindDays int `toml:"rebind_days"` +} + +type Mail struct { + // "log" prints mail to stderr (development); "smtp" sends it. + Driver string `toml:"driver"` + SMTPHost string `toml:"smtp_host"` + SMTPPort int `toml:"smtp_port"` + SMTPUser string `toml:"smtp_user"` + SMTPPass string `toml:"smtp_pass"` + // Separate senders so an invite blocklisting never touches + // verification mail. + VerifyFrom string `toml:"verify_from"` + InviteFrom string `toml:"invite_from"` + // Optional extra header for the provider's stream selection + // (e.g. Postmark's X-PM-Message-Stream). + VerifyStreamHeader string `toml:"verify_stream_header"` + InviteStreamHeader string `toml:"invite_stream_header"` +} + +type Limits struct { + PowBits int `toml:"pow_bits"` + CodesPerAddressDay int `toml:"codes_per_address_day"` + CodesPerKeyDay int `toml:"codes_per_key_day"` + CodesPerIPDay int `toml:"codes_per_ip_day"` + FirstContactsPerDay int `toml:"first_contacts_per_day"` + MailsPerSenderHour int `toml:"mails_per_sender_hour"` + MailsPerIssuerHour int `toml:"mails_per_issuer_hour"` + NewAddressDays int `toml:"new_address_days"` + NewAddressDivisor int `toml:"new_address_divisor"` + MaxMailBytes int64 `toml:"max_mail_bytes"` + QueueBytes int64 `toml:"queue_bytes"` + QueueMails int `toml:"queue_mails"` + QueueSenderShare int `toml:"queue_sender_share_percent"` + HoldDays int `toml:"hold_days"` + KnockDays int `toml:"knock_days"` + LookupsPerDay int `toml:"lookups_per_day"` + LookupBurst int `toml:"lookup_burst"` + BlockSuspendCount int `toml:"block_suspend_count"` + BlockSuspendWindowHr int `toml:"block_suspend_window_hours"` +} + +type Invite struct { + Enabled bool `toml:"enabled"` + PerRecipientDays int `toml:"per_recipient_days"` + MaxPerRecipient int `toml:"max_per_recipient"` + PerSenderDay int `toml:"per_sender_day"` + PerSenderMonth int `toml:"per_sender_month"` + GlobalPerDay int `toml:"global_per_day"` + BounceRatePct float64 `toml:"breaker_bounce_percent"` + ComplaintRatePct float64 `toml:"breaker_complaint_percent"` + BreakerWindowDays int `toml:"breaker_window_days"` + BreakerMinSample int `toml:"breaker_min_sample"` + WebhookSecret string `toml:"webhook_secret"` + DownloadURL string `toml:"download_url"` +} + +func Defaults() Config { + return Config{ + Listen: ":8080", + DataDir: "./data", + Mail: Mail{Driver: "log"}, + Limits: Limits{ + PowBits: 20, CodesPerAddressDay: 3, CodesPerKeyDay: 10, CodesPerIPDay: 20, + FirstContactsPerDay: 20, MailsPerSenderHour: 200, MailsPerIssuerHour: 20000, + NewAddressDays: 7, NewAddressDivisor: 4, + MaxMailBytes: 25 << 20, QueueBytes: 200 << 20, QueueMails: 1000, QueueSenderShare: 25, + HoldDays: 7, KnockDays: 30, LookupsPerDay: 100, LookupBurst: 20, + BlockSuspendCount: 10, BlockSuspendWindowHr: 24, + }, + Invite: Invite{ + PerRecipientDays: 30, MaxPerRecipient: 3, PerSenderDay: 5, PerSenderMonth: 20, + GlobalPerDay: 200, BounceRatePct: 2, ComplaintRatePct: 0.1, BreakerWindowDays: 7, + BreakerMinSample: 50, + }, + Push: Push{PerIPHour: 600, PerKeyHour: 120, PerTokenHour: 60, GlobalHour: 20000, RebindDays: 7}, + Share: Share{MaxBytes: 12 << 20, PerAddressDay: 30, MaxLive: 100, QuotaBytes: 500 << 20, MaxTTLDays: 30, + ReportsPerIPHour: 20}, + } +} + +func Load(path string) (*Config, error) { + c := Defaults() + if _, err := toml.DecodeFile(path, &c); err != nil { + return nil, err + } + return &c, c.Validate() +} + +func (c *Config) Validate() error { + c.URL = strings.TrimRight(c.URL, "/") + if c.URL == "" { + return errors.New("url is required") + } + if !c.Global && len(c.Domains) == 0 { + return errors.New("a private relay needs domains") + } + if c.Global && c.Closed { + return errors.New("the global relay can't be closed") + } + if !c.Closed && len(c.PartnerDomains) > 0 { + return errors.New("partner_domains only applies to a closed relay") + } + if c.Global { + c.GlobalURL = c.URL + } else if !c.Closed && (c.GlobalURL == "" || len(c.GlobalKeys) == 0) { + return errors.New("a private relay needs global_url and global_keys to accept mail from the global relay's users (or closed = true)") + } + if p := c.Push; p.Enabled && (p.KeyPath == "" || p.KeyID == "" || p.TeamID == "" || p.BundleID == "") { + return errors.New("[push] needs key_path, key_id, team_id and bundle_id") + } + if c.Share.Enabled { + if !c.Global { + return errors.New("[share] runs on the global relay only") + } + c.Share.ViewerURL = strings.TrimRight(c.Share.ViewerURL, "/") + if !strings.HasPrefix(c.Share.ViewerURL, "https://") && !strings.HasPrefix(c.Share.ViewerURL, "http://") { + return errors.New("[share] needs viewer_url") + } + } + for i, d := range c.Domains { + c.Domains[i] = strings.ToLower(strings.TrimSpace(d)) + } + for i, d := range c.PartnerDomains { + c.PartnerDomains[i] = strings.ToLower(strings.TrimSpace(d)) + } + return nil +} + +// ParseKeys parses "kid:" entries. +func ParseKeys(entries []string) (map[string]ed25519.PublicKey, error) { + out := map[string]ed25519.PublicKey{} + for _, e := range entries { + kid, pub, ok := strings.Cut(e, ":") + if !ok { + return nil, fmt.Errorf("bad key entry %q", e) + } + k, err := proto.ParseEd25519(pub) + if err != nil { + return nil, fmt.Errorf("key %s: %w", kid, err) + } + out[kid] = k + } + return out, nil +} + +func (c *Config) Write(path string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + defer f.Close() + return toml.NewEncoder(f).Encode(c) +} diff --git a/relay/internal/mailer/mailer.go b/relay/internal/mailer/mailer.go new file mode 100644 index 00000000..719dbf4b --- /dev/null +++ b/relay/internal/mailer/mailer.go @@ -0,0 +1,109 @@ +// Package mailer sends the relay's only two kinds of email: verification +// codes and invites. Each kind has its own sender and stream. +package mailer + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "log" + "net/smtp" + "strconv" + "strings" + "sync" + "time" + + "github.com/join3r/stem/relay/internal/config" +) + +type Kind int + +const ( + Verify Kind = iota + Invite +) + +type Message struct { + Kind Kind + To string + Subject string + Body string + // Tag lets bounce/complaint webhooks find the message again. + Tag string +} + +type Mailer interface { + Send(m Message) error +} + +func New(c config.Mail) Mailer { + if c.Driver == "smtp" { + return &SMTP{c: c} + } + return &Log{} +} + +// Log prints mail and keeps it, for development and tests. +type Log struct { + mu sync.Mutex + Sent []Message +} + +func (l *Log) Send(m Message) error { + l.mu.Lock() + l.Sent = append(l.Sent, m) + l.mu.Unlock() + log.Printf("mail to %s: %s\n%s", m.To, m.Subject, m.Body) + return nil +} + +func (l *Log) Messages() []Message { + l.mu.Lock() + defer l.mu.Unlock() + return append([]Message(nil), l.Sent...) +} + +type SMTP struct{ c config.Mail } + +func (s *SMTP) Send(m Message) error { + from, stream := s.c.VerifyFrom, s.c.VerifyStreamHeader + if m.Kind == Invite { + from, stream = s.c.InviteFrom, s.c.InviteStreamHeader + } + if strings.ContainsAny(m.To, "\r\n") || strings.ContainsAny(m.Subject, "\r\n") { + return fmt.Errorf("header injection") + } + var b strings.Builder + fmt.Fprintf(&b, "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n", from, m.To, m.Subject, time.Now().Format(time.RFC1123Z)) + // Gmail scores mail without a Message-ID as spam-like. + fmt.Fprintf(&b, "Message-ID: %s\r\n", messageID(envelopeFrom(from))) + fmt.Fprintf(&b, "MIME-Version: 1.0\r\nContent-Type: text/plain; charset=utf-8\r\nX-Stem-Tag: %s\r\n", m.Tag) + if stream != "" { + b.WriteString(stream + "\r\n") + } + b.WriteString("\r\n" + strings.ReplaceAll(m.Body, "\n", "\r\n")) + addr := s.c.SMTPHost + ":" + strconv.Itoa(s.c.SMTPPort) + var auth smtp.Auth + if s.c.SMTPUser != "" { + auth = smtp.PlainAuth("", s.c.SMTPUser, s.c.SMTPPass, s.c.SMTPHost) + } + return smtp.SendMail(addr, auth, envelopeFrom(from), []string{m.To}, []byte(b.String())) +} + +// envelopeFrom extracts the bare address from `Name `. +func envelopeFrom(from string) string { + if i := strings.LastIndexByte(from, '<'); i >= 0 { + return strings.TrimSuffix(from[i+1:], ">") + } + return from +} + +func messageID(from string) string { + host := "localhost" + if i := strings.LastIndexByte(from, '@'); i >= 0 { + host = from[i+1:] + } + var r [12]byte + rand.Read(r[:]) + return "<" + hex.EncodeToString(r[:]) + "@" + host + ">" +} diff --git a/relay/internal/proto/proto.go b/relay/internal/proto/proto.go new file mode 100644 index 00000000..f8e0f421 --- /dev/null +++ b/relay/internal/proto/proto.go @@ -0,0 +1,349 @@ +// Package proto holds the byte-exact parts of the relay protocol +// (PROTOCOL.md): signing strings, certificates, envelopes, sealing, +// request signatures and proof-of-work. Everything here must match +// testdata/vectors.json and Stem's TypeScript client. +package proto + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/ecdh" + "crypto/ed25519" + "crypto/hkdf" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "math/bits" + "strconv" + "strings" + "time" + +) + +// MaxSkew is how far a signed timestamp may be from the verifier's clock. +const MaxSkew = 5 * time.Minute + +// CertTTL is how long a sender certificate is valid. +const CertTTL = 24 * time.Hour + +var b64 = base64.StdEncoding + +func B64(b []byte) string { return b64.EncodeToString(b) } +func UnB64(s string) ([]byte, error) { return b64.DecodeString(s) } + +func Hex(b []byte) string { return hex.EncodeToString(b) } + +func SHA256Hex(b []byte) string { + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +// NormalizeAddress lowercases and trims; it does not validate. +func NormalizeAddress(a string) string { return strings.ToLower(strings.TrimSpace(a)) } + +// ValidAddress is a deliberately loose check: one @, non-empty local part, +// a dotted domain, no whitespace or control characters. +func ValidAddress(a string) bool { + if len(a) > 254 || a != NormalizeAddress(a) { + return false + } + at := strings.LastIndexByte(a, '@') + if at <= 0 || at == len(a)-1 || strings.Count(a, "@") != 1 { + return false + } + for _, r := range a { + if r <= ' ' || r == 0x7f { + return false + } + } + return strings.Contains(a[at+1:], ".") +} + +// Domain is the part after the last @. +func Domain(a string) string { return a[strings.LastIndexByte(a, '@')+1:] } + +// Fingerprint of an Ed25519 public key. +func Fingerprint(pub ed25519.PublicKey) string { return SHA256Hex(pub) } + +func ParseEd25519(s string) (ed25519.PublicKey, error) { + b, err := UnB64(s) + if err != nil || len(b) != ed25519.PublicKeySize { + return nil, errors.New("bad ed25519 public key") + } + return ed25519.PublicKey(b), nil +} + +// ---- certificates ---- + +type Cert struct { + V int `json:"v"` + Address string `json:"address"` + KeyFp string `json:"keyFp"` + Issuer string `json:"issuer"` + Kid string `json:"kid"` + Iat int64 `json:"iat"` + Exp int64 `json:"exp"` + Sig string `json:"sig"` +} + +func (c *Cert) SigningString() string { + return "stem-cert-v1\n" + c.Address + "\n" + c.KeyFp + "\n" + c.Issuer + "\n" + + strconv.FormatInt(c.Iat, 10) + "\n" + strconv.FormatInt(c.Exp, 10) + "\n" + c.Kid +} + +// CertString is what an envelope signature commits to. +func (c *Cert) CertString() string { return c.SigningString() + "\n" + c.Sig } + +func SignCert(c *Cert, priv ed25519.PrivateKey) { + c.Sig = B64(ed25519.Sign(priv, []byte(c.SigningString()))) +} + +// IssuerKeys resolves the relay keys allowed to certify an address +// (its domain's DNS record, or the global relay's keys). +type IssuerKeys func(address string) (issuer string, keys map[string]ed25519.PublicKey, err error) + +var ( + ErrCertIssuer = errors.New("certificate issuer is not the authority for this address") + ErrCertSig = errors.New("certificate signature invalid") + ErrCertExpired = errors.New("certificate expired or not yet valid") +) + +func VerifyCert(c *Cert, now time.Time, keys IssuerKeys) error { + if c == nil || c.V != 1 || !ValidAddress(c.Address) { + return errors.New("malformed certificate") + } + issuer, ks, err := keys(c.Address) + if err != nil { + return err + } + if issuer != c.Issuer { + return ErrCertIssuer + } + pub, ok := ks[c.Kid] + if !ok { + return ErrCertIssuer + } + sig, err := UnB64(c.Sig) + if err != nil || !ed25519.Verify(pub, []byte(c.SigningString()), sig) { + return ErrCertSig + } + ms := now.UnixMilli() + if c.Iat > ms+MaxSkew.Milliseconds() || ms >= c.Exp { + return ErrCertExpired + } + return nil +} + +// ---- envelopes ---- + +type Envelope struct { + V int `json:"v"` + ID string `json:"id"` + Ts int64 `json:"ts"` + From string `json:"from"` + To string `json:"to"` + Cert *Cert `json:"cert"` + Signer string `json:"signer"` + Sealed string `json:"sealed"` + Sig string `json:"sig"` +} + +func EnvelopeSigningString(e *Envelope, sealed []byte) string { + return "stem-env-v1\n" + e.ID + "\n" + strconv.FormatInt(e.Ts, 10) + "\n" + e.From + "\n" + e.To + "\n" + + SHA256Hex(sealed) + "\n" + SHA256Hex([]byte(e.Cert.CertString())) +} + +func SignEnvelope(e *Envelope, priv ed25519.PrivateKey) error { + sealed, err := UnB64(e.Sealed) + if err != nil { + return err + } + e.Signer = B64(priv.Public().(ed25519.PublicKey)) + e.Sig = B64(ed25519.Sign(priv, []byte(EnvelopeSigningString(e, sealed)))) + return nil +} + +func validID(id string) bool { + if len(id) != 32 { + return false + } + _, err := hex.DecodeString(id) + return err == nil && strings.ToLower(id) == id +} + +// VerifyEnvelope checks structure, certificate, binding and signature. It +// returns the decoded sealed bytes. Replay (seen ids) is the caller's job. +func VerifyEnvelope(e *Envelope, now time.Time, keys IssuerKeys) ([]byte, error) { + if e.V != 1 || !validID(e.ID) || !ValidAddress(e.From) || !ValidAddress(e.To) || e.Cert == nil { + return nil, errors.New("malformed envelope") + } + if d := now.UnixMilli() - e.Ts; d > MaxSkew.Milliseconds() || d < -MaxSkew.Milliseconds() { + return nil, errors.New("envelope timestamp out of range") + } + if e.Cert.Address != e.From { + return nil, errors.New("certificate is for a different address") + } + if err := VerifyCert(e.Cert, now, keys); err != nil { + return nil, err + } + signer, err := ParseEd25519(e.Signer) + if err != nil { + return nil, err + } + if Fingerprint(signer) != e.Cert.KeyFp { + return nil, errors.New("signer does not match certificate") + } + sealed, err := UnB64(e.Sealed) + if err != nil || len(sealed) < 32+sealOverhead { + return nil, errors.New("bad sealed payload") + } + sig, err := UnB64(e.Sig) + if err != nil || !ed25519.Verify(signer, []byte(EnvelopeSigningString(e, sealed)), sig) { + return nil, errors.New("envelope signature invalid") + } + return sealed, nil +} + +// ---- sealing ---- + +func sealAAD(id, from, to string) []byte { + return []byte("stem-seal-v1\n" + id + "\n" + from + "\n" + to) +} + +// AES-256-GCM rather than ChaCha20-Poly1305: Electron's BoringSSL (the +// desktop app's embedded server) has no ChaCha20 cipher in node:crypto. +const sealOverhead = 16 + +func sealKey(shared, ephPub, recipPub []byte) ([]byte, error) { + salt := append(append([]byte{}, ephPub...), recipPub...) + return hkdf.Key(sha256.New, shared, salt, "stem-seal-v1", 32) +} + +func sealAEAD(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} + +// Seal encrypts plaintext to an X25519 public key. ephPriv may be nil +// (random); tests pass a fixed one to produce vectors. +func Seal(recipient *ecdh.PublicKey, plaintext []byte, id, from, to string, ephPriv *ecdh.PrivateKey) ([]byte, error) { + var err error + if ephPriv == nil { + if ephPriv, err = ecdh.X25519().GenerateKey(rand.Reader); err != nil { + return nil, err + } + } + shared, err := ephPriv.ECDH(recipient) + if err != nil { + return nil, err + } + ephPub := ephPriv.PublicKey().Bytes() + key, err := sealKey(shared, ephPub, recipient.Bytes()) + if err != nil { + return nil, err + } + aead, err := sealAEAD(key) + if err != nil { + return nil, err + } + nonce := make([]byte, aead.NonceSize()) + return aead.Seal(append([]byte{}, ephPub...), nonce, plaintext, sealAAD(id, from, to)), nil +} + +// Open is Seal's inverse. The relay never calls it; it exists for tests +// and vectors. +func Open(priv *ecdh.PrivateKey, sealed []byte, id, from, to string) ([]byte, error) { + if len(sealed) < 32+sealOverhead { + return nil, errors.New("sealed payload too short") + } + eph, err := ecdh.X25519().NewPublicKey(sealed[:32]) + if err != nil { + return nil, err + } + shared, err := priv.ECDH(eph) + if err != nil { + return nil, err + } + key, err := sealKey(shared, sealed[:32], priv.PublicKey().Bytes()) + if err != nil { + return nil, err + } + aead, err := sealAEAD(key) + if err != nil { + return nil, err + } + return aead.Open(nil, make([]byte, aead.NonceSize()), sealed[32:], sealAAD(id, from, to)) +} + +// ---- request signatures ---- + +func RequestSigningString(method, pathQuery string, ts int64, body []byte) string { + return "stem-req-v1\n" + method + "\n" + pathQuery + "\n" + strconv.FormatInt(ts, 10) + "\n" + SHA256Hex(body) +} + +// VerifyRequest checks Stem-Key/Stem-Time/Stem-Sig header values. +func VerifyRequest(keyB64, tsStr, sigB64, method, pathQuery string, body []byte, now time.Time) (ed25519.PublicKey, error) { + pub, err := ParseEd25519(keyB64) + if err != nil { + return nil, err + } + ts, err := strconv.ParseInt(tsStr, 10, 64) + if err != nil { + return nil, errors.New("bad Stem-Time") + } + if d := now.UnixMilli() - ts; d > MaxSkew.Milliseconds() || d < -MaxSkew.Milliseconds() { + return nil, errors.New("Stem-Time out of range") + } + sig, err := UnB64(sigB64) + if err != nil || !ed25519.Verify(pub, []byte(RequestSigningString(method, pathQuery, ts, body)), sig) { + return nil, errors.New("bad request signature") + } + return pub, nil +} + +// ---- proof of work ---- + +func PowHash(address, keyB64, nonce string) [32]byte { + return sha256.Sum256([]byte("stem-pow-v1\n" + address + "\n" + keyB64 + "\n" + nonce)) +} + +func leadingZeroBits(h [32]byte) int { + n := 0 + for _, b := range h { + if b == 0 { + n += 8 + continue + } + return n + bits.LeadingZeros8(b) + } + return n +} + +func PowOK(address, keyB64, nonce string, bitsNeeded int) bool { + return bitsNeeded <= 0 || (len(nonce) <= 64 && leadingZeroBits(PowHash(address, keyB64, nonce)) >= bitsNeeded) +} + +// SolvePow brute-forces a nonce (clients and tests). +func SolvePow(address, keyB64 string, bitsNeeded int) string { + for i := 0; ; i++ { + n := strconv.Itoa(i) + if PowOK(address, keyB64, n, bitsNeeded) { + return n + } + } +} + +// NewID returns 32 lowercase hex characters. +func NewID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + panic(fmt.Sprintf("rand: %v", err)) + } + return Hex(b) +} diff --git a/relay/internal/proto/vectors_test.go b/relay/internal/proto/vectors_test.go new file mode 100644 index 00000000..f691bae4 --- /dev/null +++ b/relay/internal/proto/vectors_test.go @@ -0,0 +1,137 @@ +package proto + +import ( + "bytes" + "crypto/ecdh" + "crypto/ed25519" + "encoding/json" + "flag" + "os" + "path/filepath" + "testing" + "time" +) + +var update = flag.Bool("update", false, "rewrite testdata/vectors.json") + +func seed(b byte) []byte { return bytes.Repeat([]byte{b}, 32) } + +// buildVectors derives every signed or sealed structure from fixed keys. +// Ed25519 and this sealing are deterministic, so the output is stable. +func buildVectors(t *testing.T) map[string]any { + relayPriv := ed25519.NewKeyFromSeed(seed(1)) + stemPriv := ed25519.NewKeyFromSeed(seed(2)) + stemPub := stemPriv.Public().(ed25519.PublicKey) + recipEnc, err := ecdh.X25519().NewPrivateKey(seed(3)) + if err != nil { + t.Fatal(err) + } + eph, _ := ecdh.X25519().NewPrivateKey(seed(4)) + + encB64 := B64(recipEnc.PublicKey().Bytes()) + encSig := B64(ed25519.Sign(stemPriv, []byte("stem-key-v1\n"+encB64))) + + reqBody := []byte(`{"address":"martin@acme.test"}`) + reqStr := RequestSigningString("POST", "/v1/cert", 1760000000000, reqBody) + + cert := &Cert{V: 1, Address: "vlado@gmail.test", KeyFp: Fingerprint(stemPub), Issuer: "https://relay.test", + Kid: "k1", Iat: 1760000000000, Exp: 1760000000000 + CertTTL.Milliseconds()} + SignCert(cert, relayPriv) + + env := &Envelope{V: 1, ID: "0123456789abcdef0123456789abcdef", Ts: 1760000000000, + From: "vlado@gmail.test", To: "martin@acme.test", Cert: cert} + plaintext := []byte(`{"v":1,"subject":"Hi","body":"Hello Martin"}`) + sealed, err := Seal(recipEnc.PublicKey(), plaintext, env.ID, env.From, env.To, eph) + if err != nil { + t.Fatal(err) + } + env.Sealed = B64(sealed) + if err := SignEnvelope(env, stemPriv); err != nil { + t.Fatal(err) + } + + return map[string]any{ + "keys": map[string]string{ + "relaySeed": Hex(seed(1)), "relayPub": B64(relayPriv.Public().(ed25519.PublicKey)), + "stemSeed": Hex(seed(2)), "stemPub": B64(stemPub), "stemFp": Fingerprint(stemPub), + "recipientEncPriv": Hex(seed(3)), "recipientEncPub": encB64, + "ephemeralPriv": Hex(seed(4)), "ephemeralPub": B64(eph.PublicKey().Bytes()), + }, + "keyBlob": map[string]string{"enc": encB64, "encSig": encSig}, + "request": map[string]any{ + "method": "POST", "path": "/v1/cert", "time": 1760000000000, "body": string(reqBody), + "signingString": reqStr, "sig": B64(ed25519.Sign(stemPriv, []byte(reqStr))), + }, + "cert": map[string]any{"cert": cert, "signingString": cert.SigningString()}, + "envelope": map[string]any{"plaintext": string(plaintext), "envelope": env, "signingString": EnvelopeSigningString(env, sealed)}, + "pow": map[string]any{"address": "vlado@gmail.test", "key": B64(stemPub), "bits": 8, "nonce": SolvePow("vlado@gmail.test", B64(stemPub), 8)}, + } +} + +func TestVectors(t *testing.T) { + got, err := json.MarshalIndent(buildVectors(t), "", " ") + if err != nil { + t.Fatal(err) + } + got = append(got, '\n') + path := filepath.Join("..", "..", "testdata", "vectors.json") + if *update { + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%v (run go test ./internal/proto -update)", err) + } + if !bytes.Equal(got, want) { + t.Fatal("vectors changed; if intentional, rerun with -update and update the TypeScript client") + } +} + +func TestRoundTrips(t *testing.T) { + v := buildVectors(t) + env := v["envelope"].(map[string]any)["envelope"].(*Envelope) + relayPub := ed25519.NewKeyFromSeed(seed(1)).Public().(ed25519.PublicKey) + keys := func(string) (string, map[string]ed25519.PublicKey, error) { + return "https://relay.test", map[string]ed25519.PublicKey{"k1": relayPub}, nil + } + at := time.UnixMilli(env.Ts) + sealed, err := VerifyEnvelope(env, at, keys) + if err != nil { + t.Fatal(err) + } + priv, _ := ecdh.X25519().NewPrivateKey(seed(3)) + plain, err := Open(priv, sealed, env.ID, env.From, env.To) + if err != nil || string(plain) != `{"v":1,"subject":"Hi","body":"Hello Martin"}` { + t.Fatalf("open: %v %s", err, plain) + } + // Tampering with the outer addressing breaks the AAD. + if _, err := Open(priv, sealed, env.ID, env.From, "eve@acme.test"); err == nil { + t.Fatal("open succeeded with a different recipient") + } + // Expired certificate. + if _, err := VerifyEnvelope(env, at.Add(25*time.Hour), keys); err == nil { + t.Fatal("accepted with expired cert / stale timestamp") + } + // Wrong issuer. + other := func(string) (string, map[string]ed25519.PublicKey, error) { + return "https://other.test", map[string]ed25519.PublicKey{"k1": relayPub}, nil + } + if _, err := VerifyEnvelope(env, at, other); err == nil { + t.Fatal("accepted a cert from a non-authoritative issuer") + } + if !PowOK("vlado@gmail.test", v["pow"].(map[string]any)["key"].(string), v["pow"].(map[string]any)["nonce"].(string), 8) { + t.Fatal("pow") + } +} + +func TestValidAddress(t *testing.T) { + for a, want := range map[string]bool{ + "a@b.co": true, "A@b.co": false, "a@b": false, "@b.co": false, "a@@b.co": false, "a b@c.co": false, "a@b.co\n": false, + } { + if ValidAddress(a) != want { + t.Errorf("ValidAddress(%q) = %v", a, !want) + } + } +} diff --git a/relay/internal/server/invites.go b/relay/internal/server/invites.go new file mode 100644 index 00000000..16b5ca2e --- /dev/null +++ b/relay/internal/server/invites.go @@ -0,0 +1,261 @@ +package server + +import ( + "context" + "crypto/subtle" + "errors" + "fmt" + "log" + "net/http" + "strings" + "time" + + "github.com/join3r/stem/relay/internal/mailer" + "github.com/join3r/stem/relay/internal/store" +) + +var ok = map[string]string{"status": "ok"} + +// invite queues an invite. Whatever happens, the answer is the same, so +// the requester never learns whether the address is on Stem. +func (s *Server) invite(w http.ResponseWriter, r *http.Request, body []byte) error { + from, _, err := s.certified(r, body) + if err != nil { + return err + } + var in struct{ Address string } + if err := decode(body, &in); err != nil { + return err + } + to, err := addressArg(in.Address) + if err != nil { + return err + } + if served, err := s.Serves(to); err != nil { + return err + } else if !served { + return fail(421, "not_authoritative") + } + if err := s.queueInvite(from, to); err != nil { + return err + } + writeJSON(w, 202, ok) + return nil +} + +func (s *Server) queueInvite(from, to string) error { + iv, l := s.cfg.Invite, s.cfg.Limits + if !iv.Enabled || from == to { + return nil + } + // It is a knock either way. + fresh, err := s.st.Knock(from, to, s.Now().Add(-time.Duration(l.KnockDays)*day).UnixMilli()) + if err != nil || !fresh { + return err + } + if _, _, err := s.st.AddressAccount(to); err == nil { + return nil // registered: nothing to invite to + } else if !errors.Is(err, store.ErrNotFound) { + return err + } + for _, check := range []func() (bool, error){ + func() (bool, error) { return s.st.Suppressed(to) }, + func() (bool, error) { return s.st.InviterBanned(from) }, + func() (bool, error) { return s.isNewLocal(from), nil }, + } { + if bad, err := check(); err != nil || bad { + return err + } + } + if okDay, err := s.hit("invite:day:"+from, day, iv.PerSenderDay); err != nil || !okDay { + return err + } + if okMonth, err := s.hit("invite:month:"+from, 30*day, iv.PerSenderMonth); err != nil || !okMonth { + return err + } + count, _, err := s.st.InvitesTo(to, 0) + if err != nil || count >= iv.MaxPerRecipient { + return err + } + return s.st.QueueInvite(to, from) +} + +// isNewLocal: an address this relay verified within NewAddressDays. +func (s *Server) isNewLocal(addr string) bool { + _, verified, err := s.st.AddressAccount(addr) + if err != nil { + return false + } + return s.Now().UnixMilli()-verified < int64(s.cfg.Limits.NewAddressDays)*day.Milliseconds() +} + +// breakerOpen reports whether invites are paused for bounce or complaint +// rates over the window. +func (s *Server) breakerOpen() (bool, string, error) { + iv := s.cfg.Invite + since := s.Now().Add(-time.Duration(iv.BreakerWindowDays) * day).UnixMilli() + sentN, bounces, complaints, err := s.st.InviteRates(since) + if err != nil || sentN < iv.BreakerMinSample || sentN == 0 { + return false, "", err + } + br := float64(bounces) * 100 / float64(sentN) + cr := float64(complaints) * 100 / float64(sentN) + if br > iv.BounceRatePct || cr > iv.ComplaintRatePct { + return true, fmt.Sprintf("bounces %.2f%%, complaints %.3f%% of %d invites", br, cr, sentN), nil + } + return false, "", nil +} + +// DispatchInvites sends queued invites within the global budget. Called +// periodically. +func (s *Server) DispatchInvites() error { + iv := s.cfg.Invite + if !iv.Enabled { + return nil + } + if err := s.st.DropQueuedInvitesBefore(s.Now().Add(-time.Duration(iv.PerRecipientDays) * day).UnixMilli()); err != nil { + return err + } + open, why, err := s.breakerOpen() + if err != nil { + return err + } + if open { + if prev, _ := s.st.Meta("breaker"); prev != "open" { + log.Printf("ALERT: invites paused by circuit breaker: %s", why) + _ = s.st.SetMeta("breaker", "open") + } + return nil + } + _ = s.st.SetMeta("breaker", "closed") + used, err := s.st.Count("invite:global", day) + if err != nil { + return err + } + budget := iv.GlobalPerDay - used + if budget <= 0 { + return nil + } + queue, err := s.st.InviteQueue(500) + if err != nil { + return err + } + gap := time.Duration(iv.PerRecipientDays) * day + for _, q := range queue { + if budget <= 0 { + break + } + // The address may have joined, unsubscribed or been invited meanwhile. + if _, _, err := s.st.AddressAccount(q.To); err == nil { + _ = s.st.DropQueuedInvite(q.To) + continue + } + if sup, _ := s.st.Suppressed(q.To); sup { + _ = s.st.DropQueuedInvite(q.To) + continue + } + count, last, err := s.st.InvitesTo(q.To, 0) + if err != nil { + return err + } + if count >= iv.MaxPerRecipient { + _ = s.st.DropQueuedInvite(q.To) + continue + } + if last > 0 && s.Now().UnixMilli()-last < gap.Milliseconds() { + continue // batched into the next window + } + if err := s.sendInvite(q); err != nil { + log.Printf("invite to %s failed: %v", q.To, err) + continue + } + _, _ = s.hit("invite:global", day, 0) + budget-- + } + return nil +} + +func (s *Server) sendInvite(q store.QueuedInvite) error { + tag := "i-" + randomToken()[:16] + unsub := randomToken() + if err := s.st.PutUnsubToken(unsub, q.To); err != nil { + return err + } + who := q.Senders[0] + subject := who + " wants to reach you on Stem" + if n := len(q.Senders) - 1; n == 1 { + subject = who + " and 1 other want to reach you on Stem" + } else if n > 1 { + subject = fmt.Sprintf("%s and %d others want to reach you on Stem", who, n) + } + var b strings.Builder + fmt.Fprintf(&b, "%s tried to send you a message through Stem.\n\n", strings.Join(q.Senders, ", ")) + b.WriteString("Stem is a personal assistant app. Messages between Stems are end-to-end encrypted, so this email can't contain the message itself.\n\n") + if s.cfg.Invite.DownloadURL != "" { + fmt.Fprintf(&b, "Get Stem: %s\n\n", s.cfg.Invite.DownloadURL) + } + fmt.Fprintf(&b, "Never email me again: %s/v1/unsubscribe?t=%s\n", s.cfg.URL, unsub) + if err := s.mail.Send(mailer.Message{Kind: mailer.Invite, To: q.To, Subject: subject, Body: b.String(), Tag: tag}); err != nil { + return err + } + return s.st.LogInvite(q.To, tag, q.Senders) +} + +// mailEvent takes bounce and complaint notifications from the mail +// provider (normalized by whatever forwards them). +func (s *Server) mailEvent(w http.ResponseWriter, r *http.Request, body []byte) error { + secret := s.cfg.Invite.WebhookSecret + if secret == "" || subtle.ConstantTimeCompare([]byte(r.Header.Get("X-Stem-Webhook-Secret")), []byte(secret)) != 1 { + return fail(401, "bad_secret") + } + var in struct{ Tag, Kind string } + if err := decode(body, &in); err != nil { + return err + } + if in.Kind != "bounce" && in.Kind != "complaint" { + return fail(400, "bad_kind") + } + if err := s.st.LogMailEvent(in.Tag, in.Kind); err != nil { + return err + } + to, senders, err := s.st.InviteTagAddress(in.Tag) + if err == nil { + if err := s.st.Suppress(to); err != nil { + return err + } + if in.Kind == "complaint" { + for _, f := range senders { + if err := s.st.BanInviter(f); err != nil { + return err + } + } + } + } else if !errors.Is(err, store.ErrNotFound) { + return err + } + writeJSON(w, 200, map[string]any{}) + return nil +} + +// Run does periodic work until ctx ends: sweeping and invite dispatch. +func (s *Server) Run(ctx context.Context) { + t := time.NewTicker(time.Minute) + defer t.Stop() + sweep := time.Now() + for { + if time.Since(sweep) >= 10*time.Minute { + if err := s.st.Sweep(s.cfg.Limits.HoldDays); err != nil { + log.Printf("sweep: %v", err) + } + sweep = time.Now() + } + if err := s.DispatchInvites(); err != nil { + log.Printf("invites: %v", err) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} diff --git a/relay/internal/server/mail.go b/relay/internal/server/mail.go new file mode 100644 index 00000000..86dbf4c5 --- /dev/null +++ b/relay/internal/server/mail.go @@ -0,0 +1,316 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/join3r/stem/relay/internal/proto" + "github.com/join3r/stem/relay/internal/store" +) + +// ---- delivery ---- + +var sent = map[string]string{"status": "sent"} + +func (s *Server) postMail(w http.ResponseWriter, r *http.Request, body []byte) error { + var env proto.Envelope + if err := decode(body, &env); err != nil { + return err + } + env.From, env.To = proto.NormalizeAddress(env.From), proto.NormalizeAddress(env.To) + sealed, err := proto.VerifyEnvelope(&env, s.Now(), s.issuerKeys) + if errors.Is(err, errOutsider) { + return errOutsider + } else if err != nil { + return fail(400, "bad_envelope") + } + if int64(len(sealed)) > s.cfg.Limits.MaxMailBytes { + return fail(413, "too_large") + } + if ok, err := s.Serves(env.To); err != nil { + return err + } else if !ok { + return fail(421, "not_authoritative") + } + if seen, err := s.st.Seen(env.ID); err != nil { + return err + } else if seen { + return fail(409, "duplicate") + } + signerFp := env.Cert.KeyFp + drop := func() error { + if err := s.st.MarkSeen(env.ID, signerFp, env.From, env.To, "dropped"); err != nil { + return err + } + writeJSON(w, 202, sent) + return nil + } + + l := s.cfg.Limits + if sus, err := s.st.Suspended(env.From); err != nil { + return err + } else if sus { + return drop() + } + if ok, err := s.hit("send:hour:"+env.From, time.Hour, s.scaled(env.From, l.MailsPerSenderHour)); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + if ok, err := s.hit("issuer:hour:"+env.Cert.Issuer, time.Hour, l.MailsPerIssuerHour); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + + acc, _, err := s.st.AddressAccount(env.To) + if errors.Is(err, store.ErrNotFound) || err == nil && acc.Revoked { + return drop() + } else if err != nil { + return err + } + if blocked, err := s.st.Blocks(acc.Fp, env.From); err != nil { + return err + } else if blocked { + return drop() + } + allowed, err := s.st.Allows(acc.Fp, env.From) + if err != nil { + return err + } + if !allowed { + knockSince := s.Now().Add(-time.Duration(l.KnockDays) * day).UnixMilli() + fresh, err := s.st.Knock(env.From, env.To, knockSince) + if err != nil { + return err + } + if !fresh { + return drop() + } + if ok, err := s.hit("first:day:"+env.From, day, s.scaled(env.From, l.FirstContactsPerDay)); err != nil { + return err + } else if !ok { + return drop() + } + } + + bytes, mails, err := s.st.QueueUsage(env.To) + if err != nil { + return err + } + size := int64(len(body)) + full := bytes+size > l.QueueBytes || mails+1 > l.QueueMails + if !full && l.QueueSenderShare > 0 && l.QueueSenderShare < 100 { + fb, fm, err := s.st.QueueUsageFrom(env.To, env.From) + if err != nil { + return err + } + full = fb+size > l.QueueBytes*int64(l.QueueSenderShare)/100 || fm+1 > l.QueueMails*l.QueueSenderShare/100 + } + if full { + if allowed { + return fail(507, "queue_full") + } + return drop() + } + + expires := s.Now().Add(time.Duration(l.HoldDays) * day).UnixMilli() + if err := s.st.Hold(store.Held{ID: env.ID, To: env.To, From: env.From, Size: size, Envelope: body}, expires); err != nil { + return err + } + if err := s.st.MarkSeen(env.ID, signerFp, env.From, env.To, "pending"); err != nil { + return err + } + s.hub.publish(acc.Fp, store.Held{ID: env.ID, To: env.To, From: env.From, Size: size, Envelope: body}) + writeJSON(w, 202, sent) + return nil +} + +// scaled divides a limit for addresses this relay verified recently. +// Senders on other relays are judged by their own relay. +func (s *Server) scaled(from string, limit int) int { + l := s.cfg.Limits + if limit <= 0 || l.NewAddressDivisor <= 1 { + return limit + } + _, verified, err := s.st.AddressAccount(from) + if err != nil { + return limit + } + if s.Now().UnixMilli()-verified < int64(l.NewAddressDays)*day.Milliseconds() { + return max(1, limit/l.NewAddressDivisor) + } + return limit +} + +func (s *Server) mailStatus(w http.ResponseWriter, r *http.Request, body []byte) error { + _, fp, err := s.certified(r, body) + if err != nil { + return err + } + signerFp, from, to, status, err := s.st.SeenRow(r.URL.Query().Get("id")) + if errors.Is(err, store.ErrNotFound) || err == nil && signerFp != fp { + writeJSON(w, 200, sent) + return nil + } else if err != nil { + return err + } + acc, _, err := s.st.AddressAccount(to) + if err != nil || status == "dropped" { + writeJSON(w, 200, sent) + return nil + } + if ok, err := s.st.Allows(acc.Fp, from); err != nil { + return err + } else if !ok { + writeJSON(w, 200, sent) + return nil + } + writeJSON(w, 200, map[string]string{"status": status}) + return nil +} + +func (s *Server) ack(w http.ResponseWriter, r *http.Request, body []byte) error { + _, addrs, err := s.own(r, body) + if err != nil { + return err + } + var in struct { + IDs []string `json:"ids"` + } + if err := decode(body, &in); err != nil { + return err + } + if len(in.IDs) > 1000 { + return fail(400, "too_many") + } + n, err := s.st.Ack(in.IDs, addrs) + if err != nil { + return err + } + writeJSON(w, 200, map[string]int{"acked": n}) + return nil +} + +// ---- stream ---- + +func (s *Server) stream(w http.ResponseWriter, r *http.Request) { + acc, addrs, err := s.own(r, nil) + if err != nil { + var ae *apiError + if errors.As(err, &ae) { + writeJSON(w, ae.status, map[string]string{"error": ae.code}) + } else { + writeJSON(w, 500, map[string]string{"error": "internal"}) + } + return + } + if !s.streams.take(acc.Fp, streamsPerKey) { + writeJSON(w, 429, map[string]string{"error": "too_many_streams"}) + return + } + defer s.streams.give(acc.Fp) + flusher, ok := w.(http.Flusher) + if !ok { + writeJSON(w, 500, map[string]string{"error": "no_streaming"}) + return + } + ch := s.hub.subscribe(acc.Fp) + defer s.hub.unsubscribe(ch) + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Accel-Buffering", "no") + w.WriteHeader(200) + + sentIDs := map[string]bool{} + send := func(h store.Held) bool { + if sentIDs[h.ID] { + return true + } + sentIDs[h.ID] = true + _, err := fmt.Fprintf(w, "event: mail\nid: %s\ndata: %s\n\n", h.ID, compact(h.Envelope)) + flusher.Flush() + return err == nil + } + held, err := s.st.HeldFor(addrs) + if err != nil { + return + } + for _, h := range held { + if !send(h) { + return + } + } + fmt.Fprint(w, ": ready\n\n") + flusher.Flush() + + ping := time.NewTicker(25 * time.Second) + defer ping.Stop() + for { + select { + case <-r.Context().Done(): + return + case h := <-ch: + if !send(h) { + return + } + case <-ping.C: + if _, err := fmt.Fprint(w, ": ping\n\n"); err != nil { + return + } + flusher.Flush() + } + } +} + +// compact keeps an SSE data line on one line. +func compact(b []byte) []byte { + var v json.RawMessage = b + out, err := json.Marshal(v) + if err != nil { + return b + } + return out +} + +// hub fans new mail out to open streams by account fingerprint, not by +// address: when an address moves to a new key, the old key's open stream +// must stop receiving it at once. +type hub struct { + mu sync.Mutex + subs map[chan store.Held]string +} + +func newHub() *hub { return &hub{subs: map[chan store.Held]string{}} } + +func (h *hub) subscribe(fp string) chan store.Held { + ch := make(chan store.Held, 64) + h.mu.Lock() + h.subs[ch] = fp + h.mu.Unlock() + return ch +} + +func (h *hub) unsubscribe(ch chan store.Held) { + h.mu.Lock() + delete(h.subs, ch) + h.mu.Unlock() +} + +func (h *hub) publish(fp string, m store.Held) { + h.mu.Lock() + defer h.mu.Unlock() + for ch, sub := range h.subs { + if sub == fp { + select { + case ch <- m: + default: // slow reader: it gets it on reconnect + } + } + } +} diff --git a/relay/internal/server/push.go b/relay/internal/server/push.go new file mode 100644 index 00000000..811d54b1 --- /dev/null +++ b/relay/internal/server/push.go @@ -0,0 +1,170 @@ +package server + +import ( + "context" + "encoding/json" + "log" + "net/http" + "regexp" + "time" + + "github.com/join3r/stem/relay/internal/apns" + "github.com/join3r/stem/relay/internal/proto" +) + +// ---- push gateway ---- +// +// A Stem without the app's APNs key sends {token, env, wake} signed with its +// own key; the relay writes the alert itself from fixed wording, so no Stem +// can put its own words on someone's lock screen, and sends it with the +// Awantech .p8. A token belongs to the first key that pushes to it. + +// Pusher is what the gateway sends through: *apns.Client, or a test's. +type Pusher interface { + Send(ctx context.Context, env, deviceToken string, payload []byte) (apns.Result, string) +} + +type Wake struct { + Kind string `json:"kind"` + ApprovalKind string `json:"approvalKind,omitempty"` + ConversationID string `json:"conversationId,omitempty"` + ThreadID string `json:"threadId,omitempty"` + ApprovalID string `json:"approvalId,omitempty"` + TaskID string `json:"taskId,omitempty"` + Failed bool `json:"failed,omitempty"` +} + +var ( + deviceTokenRe = regexp.MustCompile(`^[0-9a-f]{64,200}$`) + pushIDRe = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`) +) + +func idOK(id string) bool { return id == "" || pushIDRe.MatchString(id) } + +// Phrasing mirrors src/server/push/index.ts without the label: the relay +// never shows text a Stem chose. +func phrasing(w Wake) (title, body string, ok bool) { + switch w.Kind { + case "approval": + switch w.ApprovalKind { + case "exec": + return "Approval needed", "A command is waiting for your decision.", true + case "mcp": + return "Approval needed", "An MCP change is waiting for your decision.", true + case "instructions": + return "Approval needed", "A change to your instructions is waiting.", true + case "harness": + return "Approval needed", "A coding agent is waiting for your decision.", true + case "skill", "": + return "Approval needed", "A skill is waiting for your review.", true + } + case "turn": + if w.Failed { + return "A turn stopped", "A turn you started did not finish.", true + } + return "Stem finished", "A turn you started has finished.", true + case "task": + return "Task update", "A scheduled task has something for you.", true + case "mail": + return "New mail", "You have new mail in Stem.", true + } + return "", "", false +} + +func pushPayload(w Wake) ([]byte, bool) { + title, body, ok := phrasing(w) + if !ok || !idOK(w.ConversationID) || !idOK(w.ThreadID) || !idOK(w.ApprovalID) || !idOK(w.TaskID) { + return nil, false + } + aps := map[string]any{"alert": map[string]string{"title": title, "body": body}, "sound": "default"} + if w.ConversationID != "" { + aps["thread-id"] = "mail:" + w.ConversationID + } else if w.ThreadID != "" { + aps["thread-id"] = w.ThreadID + } + stem := map[string]any{"kind": w.Kind} + for k, v := range map[string]string{"conversationId": w.ConversationID, "approvalKind": w.ApprovalKind, + "threadId": w.ThreadID, "approvalId": w.ApprovalID, "taskId": w.TaskID} { + if v != "" { + stem[k] = v + } + } + if w.Kind == "turn" { + stem["failed"] = w.Failed + } + b, _ := json.Marshal(map[string]any{"aps": aps, "stem": stem}) + return b, true +} + +func (s *Server) push(w http.ResponseWriter, r *http.Request, body []byte) error { + if s.Pusher == nil { + return fail(404, "push_disabled") + } + pub, err := s.signed(r, body) + if err != nil { + return err + } + var in struct { + Token string `json:"token"` + Env string `json:"env"` + Wake Wake `json:"wake"` + } + if err := decode(body, &in); err != nil { + return err + } + if in.Env == "" { + in.Env = "production" + } + if !deviceTokenRe.MatchString(in.Token) || (in.Env != "production" && in.Env != "sandbox") { + return fail(400, "bad_token") + } + payload, ok := pushPayload(in.Wake) + if !ok { + return fail(400, "bad_wake") + } + p := s.cfg.Push + fp := proto.Fingerprint(pub) + // Keys are free, so the per-key ceiling alone stops nobody: the client IP + // is counted first. The token's ceiling comes after ownership, so a key + // that does not own a token can't spend its allowance, and the global one + // last, so refused requests never eat everyone else's. + limit := func(k string, n int) error { + if ok, err := s.hit(k, time.Hour, n); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + return nil + } + if err := limit("push:ip:"+ipBucket(s.clientIP(r)), p.PerIPHour); err != nil { + return err + } + if err := limit("push:key:"+fp, p.PerKeyHour); err != nil { + return err + } + if owns, err := s.st.ClaimPushToken(in.Token, fp, time.Duration(p.RebindDays)*day); err != nil { + return err + } else if !owns { + return fail(409, "token_bound") + } + if err := limit("push:tok:"+in.Token, p.PerTokenHour); err != nil { + return err + } + if err := limit("push:global", p.GlobalHour); err != nil { + return err + } + res, detail := s.Pusher.Send(r.Context(), in.Env, in.Token, payload) + switch res { + case apns.Sent: + writeJSON(w, 200, map[string]string{"result": "sent"}) + case apns.Gone: + if err := s.st.DropPushToken(in.Token); err != nil { + return err + } + writeJSON(w, 200, map[string]string{"result": "gone"}) + default: + log.Printf("push: apns refused: %s", detail) + return fail(502, "apns_failed") + } + return nil +} diff --git a/relay/internal/server/push_test.go b/relay/internal/server/push_test.go new file mode 100644 index 00000000..b6cd0ac3 --- /dev/null +++ b/relay/internal/server/push_test.go @@ -0,0 +1,99 @@ +package server + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + + "github.com/join3r/stem/relay/internal/apns" +) + +type fakePusher struct { + mu sync.Mutex + sent []string // token|payload + next apns.Result +} + +func (f *fakePusher) Send(_ context.Context, env, token string, payload []byte) (apns.Result, string) { + f.mu.Lock() + defer f.mu.Unlock() + f.sent = append(f.sent, env+"|"+token+"|"+string(payload)) + return f.next, "" +} + +var tokA = strings.Repeat("ab", 32) + +func TestPushGateway(t *testing.T) { + w := newWorld(t) + g := w.global + a := w.newStem(g, "a@gmail.test") + b := w.newStem(g, "b@gmail.test") + + if code, _ := a.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokA, "wake": map[string]any{"kind": "mail"}}, false); code != 404 { + t.Fatalf("disabled gateway answered %d", code) + } + fp := &fakePusher{} + g.srv.Pusher = fp + + code, out := a.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokA, + "wake": map[string]any{"kind": "approval", "approvalKind": "exec", "threadId": "t-1", "approvalId": "42", "label": "rm -rf secrets"}}, false) + if code != 200 || !strings.Contains(string(out), `"sent"`) { + t.Fatalf("push: %d %s", code, out) + } + parts := strings.SplitN(fp.sent[0], "|", 3) + if parts[0] != "production" || parts[1] != tokA { + t.Fatalf("sent %v", parts[:2]) + } + var p struct { + Aps struct { + Alert struct{ Title, Body string } + } + Stem map[string]any + } + _ = json.Unmarshal([]byte(parts[2]), &p) + if p.Aps.Alert.Body != "A command is waiting for your decision." || p.Stem["approvalId"] != "42" || strings.Contains(parts[2], "secrets") { + t.Fatalf("payload %s", parts[2]) + } + + // The token is a's now: b may not push to it. + if code, out := b.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokA, "wake": map[string]any{"kind": "mail"}}, false); code != 409 { + t.Fatalf("other key: %d %s", code, out) + } + // Unknown kinds and odd ids are refused, never phrased. + for _, wake := range []map[string]any{{"kind": "custom"}, {"kind": "mail", "conversationId": "hi"}} { + if code, _ := a.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokA, "wake": wake}, false); code != 400 { + t.Fatalf("wake %v answered %d", wake, code) + } + } + if code, _ := a.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": "nothex", "wake": map[string]any{"kind": "mail"}}, false); code != 400 { + t.Fatal("bad token accepted") + } + + // Apple says gone: the binding is dropped, so b can claim the token. + fp.next = apns.Gone + if _, out := a.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokA, "wake": map[string]any{"kind": "mail"}}, false); !strings.Contains(string(out), `"gone"`) { + t.Fatalf("gone: %s", out) + } + fp.next = apns.Sent + if code, _ := b.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokA, "env": "sandbox", "wake": map[string]any{"kind": "turn", "failed": true}}, false); code != 200 { + t.Fatalf("b after gone: %d", code) + } + + // Per-IP ceiling: fresh keys from one address don't get around it. + g.srv.cfg.Push.PerIPHour = 4 // four pushes from this IP so far + c := w.newStem(g, "c@gmail.test") + if code, _ := c.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": strings.Repeat("ef", 32), "wake": map[string]any{"kind": "mail"}}, false); code != 429 { + t.Fatalf("per-IP limit: %d", code) + } + g.srv.cfg.Push.PerIPHour = 600 + + // Per-token ceiling. + g.srv.cfg.Push.PerTokenHour = 1 + tokB := strings.Repeat("cd", 32) + a.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokB, "wake": map[string]any{"kind": "mail"}}, false) + if code, _ := a.do(g.http.URL, "POST", "/v1/push", map[string]any{"token": tokB, "wake": map[string]any{"kind": "mail"}}, false); code != 429 { + t.Fatalf("rate limit: %d", code) + } +} diff --git a/relay/internal/server/server.go b/relay/internal/server/server.go new file mode 100644 index 00000000..d46db531 --- /dev/null +++ b/relay/internal/server/server.go @@ -0,0 +1,854 @@ +// Package server is the relay's HTTP API (PROTOCOL.md). +package server + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "html" + "io" + "log" + "math/big" + "net" + "net/http" + "net/netip" + "net/url" + "slices" + "strings" + "sync" + "time" + + "github.com/join3r/stem/relay/internal/authority" + "github.com/join3r/stem/relay/internal/config" + "github.com/join3r/stem/relay/internal/mailer" + "github.com/join3r/stem/relay/internal/proto" + "github.com/join3r/stem/relay/internal/store" +) + +const Version = "1.0.0" + +type Server struct { + cfg *config.Config + st *store.Store + mail mailer.Mailer + resolver *authority.Resolver + Now func() time.Time + + keysMu sync.RWMutex + signKid string + signKey ed25519.PrivateKey + pubKeys map[string]ed25519.PublicKey + + hub *hub + mailSlots chan struct{} + ipSlots counted + streams counted + + // Pusher is set when [push] is enabled; nil answers push_disabled. + Pusher Pusher +} + +func New(cfg *config.Config, st *store.Store, m mailer.Mailer, res *authority.Resolver) (*Server, error) { + s := &Server{cfg: cfg, st: st, mail: m, resolver: res, Now: time.Now, hub: newHub(), mailSlots: make(chan struct{}, 32)} + if err := s.LoadKeys(); err != nil { + return nil, err + } + if cfg.Global { + res.GlobalURL = cfg.URL + res.GlobalKeys = s.PublicKeys() + } + return s, nil +} + +// LoadKeys reads the relay's signing keys, creating the first one. +func (s *Server) LoadKeys() error { + keys, err := s.st.RelayKeys() + if err != nil { + return err + } + if len(keys) == 0 { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + if err := s.st.AddRelayKey("k1", priv, true); err != nil { + return err + } + keys, _ = s.st.RelayKeys() + } + pubs := map[string]ed25519.PublicKey{} + var kid string + var priv ed25519.PrivateKey + for _, k := range keys { + p := ed25519.PrivateKey(k.Priv) + pubs[k.Kid] = p.Public().(ed25519.PublicKey) + if k.Active { + kid, priv = k.Kid, p + } + } + if priv == nil { + return errors.New("no active relay key") + } + s.keysMu.Lock() + s.signKid, s.signKey, s.pubKeys = kid, priv, pubs + s.keysMu.Unlock() + if s.cfg.Global && s.resolver != nil { + s.resolver.GlobalKeys = pubs + } + return nil +} + +func (s *Server) PublicKeys() map[string]ed25519.PublicKey { + s.keysMu.RLock() + defer s.keysMu.RUnlock() + return s.pubKeys +} + +// Serves reports whether this relay is the authority for an address. +func (s *Server) Serves(address string) (bool, error) { + d := proto.Domain(address) + if !s.cfg.Global { + return slices.Contains(s.cfg.Domains, d), nil + } + rec, err := s.resolver.DomainRecord(d) + if err != nil { + return false, err + } + return rec == nil, nil +} + +// errOutsider: the sender is outside a closed relay's circle, or belongs to a +// closed domain this relay doesn't serve. +var errOutsider = fail(403, "closed_relay") + +// outsider reports whether certificates for the address must be refused: +// on a closed relay, anyone outside its domains and partners; on any relay, +// the people of a domain whose record says access=closed, unless we serve it. +func (s *Server) outsider(address string) (bool, error) { + d := proto.Domain(address) + if slices.Contains(s.cfg.Domains, d) { + return false, nil + } + if s.cfg.Closed { + return !slices.Contains(s.cfg.PartnerDomains, d), nil + } + rec, err := s.resolver.DomainRecord(d) + if err != nil { + return false, err + } + return rec != nil && rec.Closed, nil +} + +// issuerKeys verifies certificates: our own keys for our addresses, DNS or +// the global relay for everyone else. Outsiders (see outsider) get none. +func (s *Server) issuerKeys(address string) (string, map[string]ed25519.PublicKey, error) { + if out, err := s.outsider(address); err != nil { + return "", nil, err + } else if out { + return "", nil, errOutsider + } + if ok, err := s.Serves(address); err != nil { + return "", nil, err + } else if ok { + return s.cfg.URL, s.PublicKeys(), nil + } + return s.resolver.Authority(address) +} + +func (s *Server) issueCert(address, fp string) *proto.Cert { + s.keysMu.RLock() + kid, priv := s.signKid, s.signKey + s.keysMu.RUnlock() + t := s.Now() + c := &proto.Cert{V: 1, Address: address, KeyFp: fp, Issuer: s.cfg.URL, Kid: kid, + Iat: t.UnixMilli(), Exp: t.Add(proto.CertTTL).UnixMilli()} + proto.SignCert(c, priv) + return c +} + +// ---- HTTP plumbing ---- + +type apiError struct { + status int + code string +} + +func (e *apiError) Error() string { return e.code } + +func fail(status int, code string) error { return &apiError{status, code} } + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +type handler func(w http.ResponseWriter, r *http.Request, body []byte) error + +func (s *Server) wrap(maxBody int64, h handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var body []byte + if r.Body != nil { + var err error + body, err = io.ReadAll(io.LimitReader(r.Body, maxBody+1)) + if err != nil { + writeJSON(w, 400, map[string]string{"error": "bad_body"}) + return + } + if int64(len(body)) > maxBody { + writeJSON(w, 413, map[string]string{"error": "too_large"}) + return + } + } + if err := h(w, r, body); err != nil { + var ae *apiError + if errors.As(err, &ae) { + writeJSON(w, ae.status, map[string]string{"error": ae.code}) + return + } + log.Printf("%s %s: %v", r.Method, r.URL.Path, err) + writeJSON(w, 500, map[string]string{"error": "internal"}) + } + } +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + small := int64(64 << 10) + envMax := s.cfg.Limits.MaxMailBytes*4/3 + 16<<10 + mux.HandleFunc("GET /.well-known/stem-relay", s.wrap(0, s.wellKnown)) + mux.HandleFunc("GET /v1/info", s.wrap(0, s.info)) + mux.HandleFunc("POST /v1/register", s.wrap(small, s.register)) + mux.HandleFunc("POST /v1/verify", s.wrap(small, s.verify)) + mux.HandleFunc("GET /v1/verify-link", s.wrap(0, s.verifyLinkPage)) + mux.HandleFunc("POST /v1/verify-link", s.wrap(small, s.verifyLink)) + mux.HandleFunc("POST /v1/cert", s.wrap(small, s.cert)) + mux.HandleFunc("GET /v1/keys", s.wrap(0, s.keys)) + mux.HandleFunc("GET /v1/keystatus", s.wrap(0, s.keystatus)) + mux.HandleFunc("POST /v1/mail", s.limited(s.mailSlots, s.wrap(envMax, s.postMail))) + mux.HandleFunc("GET /v1/mail/status", s.wrap(0, s.mailStatus)) + mux.HandleFunc("GET /v1/stream", s.stream) + mux.HandleFunc("POST /v1/ack", s.wrap(small, s.ack)) + mux.HandleFunc("POST /v1/allow", s.wrap(small, s.listOp("allow"))) + mux.HandleFunc("POST /v1/unallow", s.wrap(small, s.listOp("unallow"))) + mux.HandleFunc("POST /v1/block", s.wrap(small, s.listOp("block"))) + mux.HandleFunc("POST /v1/unblock", s.wrap(small, s.listOp("unblock"))) + mux.HandleFunc("PUT /v1/account", s.wrap(small, s.account)) + mux.HandleFunc("POST /v1/invite", s.wrap(small, s.invite)) + mux.HandleFunc("POST /v1/revoke", s.wrap(small, s.revoke)) + mux.HandleFunc("GET /v1/unsubscribe", s.wrap(0, s.unsubscribePage)) + mux.HandleFunc("POST /v1/unsubscribe", s.wrap(small, s.unsubscribe)) + mux.HandleFunc("POST /v1/hooks/mail-event", s.wrap(small, s.mailEvent)) + mux.HandleFunc("POST /v1/push", s.wrap(small, s.push)) + mux.HandleFunc("POST /v1/shares", s.limited(s.mailSlots, s.wrap(s.cfg.Share.MaxBytes, s.postShare))) + mux.HandleFunc("GET /v1/shares", s.wrap(0, s.listShares)) + mux.HandleFunc("GET /v1/shares/{id}", s.wrap(0, s.getShare)) + mux.HandleFunc("DELETE /v1/shares/{id}", s.wrap(0, s.deleteShare)) + mux.HandleFunc("POST /v1/shares/{id}/report", s.wrap(small, s.reportShare)) + mux.HandleFunc("GET /s/{id}", s.wrap(0, s.sharePage)) + mux.HandleFunc("GET /assets/", s.viewerAsset) + mux.HandleFunc("GET /og.png", s.viewerAsset) + mux.HandleFunc("GET /favicon.png", s.viewerAsset) + return mux +} + +// limited caps concurrent requests to a handler: POST /v1/mail is +// unauthenticated until its envelope verifies and reads up to the mail size +// limit into memory. +// +// One client can't hog the slots: at most perIP at a time from one address, +// and the body must arrive within mailReadDeadline, so slow uploads give +// their slot back quickly. +func (s *Server) limited(slots chan struct{}, h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ip := ipBucket(s.clientIP(r)) + if !s.ipSlots.take(ip, perIPMail) { + writeJSON(w, 429, map[string]string{"error": "rate_limited"}) + return + } + defer s.ipSlots.give(ip) + _ = http.NewResponseController(w).SetReadDeadline(time.Now().Add(mailReadDeadline)) + select { + case slots <- struct{}{}: + defer func() { <-slots }() + h(w, r) + case <-time.After(5 * time.Second): + writeJSON(w, 503, map[string]string{"error": "busy"}) + } + } +} + +const ( + perIPMail = 4 + mailReadDeadline = 30 * time.Second + streamsPerKey = 5 +) + +// counted is a concurrency counter per key (client IP, account). +type counted struct { + mu sync.Mutex + n map[string]int +} + +func (c *counted) take(k string, max int) bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.n == nil { + c.n = map[string]int{} + } + if c.n[k] >= max { + return false + } + c.n[k]++ + return true +} + +func (c *counted) give(k string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.n[k]--; c.n[k] <= 0 { + delete(c.n, k) + } +} + +// ipBucket is what per-IP limits count: the address itself for IPv4, its +// /64 for IPv6, since one client holds a whole /64 and can rotate through it. +func ipBucket(ip string) string { + a, err := netip.ParseAddr(ip) + if err != nil || a.Unmap().Is4() { + return ip + } + p, _ := a.Prefix(64) + return p.String() +} + +func (s *Server) clientIP(r *http.Request) string { + if s.cfg.TrustProxy { + // The rightmost entry is the one our own proxy appended; anything + // left of it came from the client and could be forged. + if xff := r.Header.Values("X-Forwarded-For"); len(xff) > 0 { + parts := strings.Split(xff[len(xff)-1], ",") + if ip := strings.TrimSpace(parts[len(parts)-1]); ip != "" { + return ip + } + } + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// signed verifies the Stem-* request headers and returns the key. +func (s *Server) signed(r *http.Request, body []byte) (ed25519.PublicKey, error) { + pub, err := proto.VerifyRequest(r.Header.Get("Stem-Key"), r.Header.Get("Stem-Time"), r.Header.Get("Stem-Sig"), + r.Method, r.URL.RequestURI(), body, s.Now()) + if err != nil { + return nil, fail(401, "bad_signature") + } + return pub, nil +} + +// own verifies a signed request from a key registered here. +func (s *Server) own(r *http.Request, body []byte) (*store.Account, []string, error) { + pub, err := s.signed(r, body) + if err != nil { + return nil, nil, err + } + acc, err := s.st.Account(proto.Fingerprint(pub)) + if errors.Is(err, store.ErrNotFound) || err == nil && acc.Revoked { + return nil, nil, fail(403, "not_registered") + } else if err != nil { + return nil, nil, err + } + addrs, err := s.st.AccountAddresses(acc.Fp) + return acc, addrs, err +} + +// certified verifies a signed request carrying Stem-Cert and returns the +// certified address and the key's fingerprint. +func (s *Server) certified(r *http.Request, body []byte) (string, string, error) { + pub, err := s.signed(r, body) + if err != nil { + return "", "", err + } + raw, err := proto.UnB64(r.Header.Get("Stem-Cert")) + if err != nil { + return "", "", fail(401, "bad_certificate") + } + var c proto.Cert + if json.Unmarshal(raw, &c) != nil { + return "", "", fail(401, "bad_certificate") + } + if err := proto.VerifyCert(&c, s.Now(), s.issuerKeys); err != nil { + if errors.Is(err, errOutsider) { + return "", "", errOutsider + } + return "", "", fail(401, "bad_certificate") + } + fp := proto.Fingerprint(pub) + if c.KeyFp != fp { + return "", "", fail(401, "bad_certificate") + } + if sus, err := s.st.Suspended(c.Address); err != nil { + return "", "", err + } else if sus { + return "", "", fail(403, "suspended") + } + return c.Address, fp, nil +} + +func decode(body []byte, v any) error { + if err := json.Unmarshal(body, v); err != nil { + return fail(400, "bad_json") + } + return nil +} + +func addressArg(a string) (string, error) { + a = proto.NormalizeAddress(a) + if !proto.ValidAddress(a) { + return "", fail(400, "bad_address") + } + return a, nil +} + +func (s *Server) hit(k string, window time.Duration, limit int) (bool, error) { + return s.st.Hit(k, window, limit) +} + +const day = 24 * time.Hour + +// ---- discovery ---- + +func (s *Server) wellKnown(w http.ResponseWriter, r *http.Request, _ []byte) error { + type key struct { + Kid string `json:"kid"` + Pub string `json:"pub"` + } + var keys []key + for kid, pub := range s.PublicKeys() { + keys = append(keys, key{kid, proto.B64(pub)}) + } + slices.SortFunc(keys, func(a, b key) int { return strings.Compare(a.Kid, b.Kid) }) + writeJSON(w, 200, map[string]any{"v": 1, "url": s.cfg.URL, "global": s.cfg.Global, "keys": keys}) + return nil +} + +func (s *Server) info(w http.ResponseWriter, r *http.Request, _ []byte) error { + features := []string{"pow", "unlisted", "status"} + if s.cfg.Invite.Enabled { + features = append(features, "invites") + } + if s.Pusher != nil { + features = append(features, "push") + } + l := s.cfg.Limits + limits := map[string]any{"maxMailBytes": l.MaxMailBytes, "powBits": l.PowBits, "holdDays": l.HoldDays} + if s.cfg.Share.Enabled { + features = append(features, "share") + limits["maxShareBytes"] = s.cfg.Share.MaxBytes + limits["maxShareTtlDays"] = s.cfg.Share.MaxTTLDays + } + writeJSON(w, 200, map[string]any{"v": 1, "version": Version, "features": features, "limits": limits}) + return nil +} + +// ---- register / verify ---- + +func randomDigits(n int) string { + var b strings.Builder + for range n { + d, _ := rand.Int(rand.Reader, big.NewInt(10)) + b.WriteByte(byte('0' + d.Int64())) + } + return b.String() +} + +func randomToken() string { + b := make([]byte, 24) + _, _ = rand.Read(b) + return strings.TrimRight(strings.NewReplacer("+", "-", "/", "_").Replace(proto.B64(b)), "=") +} + +func codeHash(code, address, fp string) string { + return proto.SHA256Hex([]byte("stem-code-v1\n" + code + "\n" + address + "\n" + fp)) +} + +func (s *Server) register(w http.ResponseWriter, r *http.Request, body []byte) error { + pub, err := s.signed(r, body) + if err != nil { + return err + } + var in struct { + Address string `json:"address"` + KeyBlob string `json:"keyBlob"` + Pow struct { + Nonce string `json:"nonce"` + } `json:"pow"` + } + if err := decode(body, &in); err != nil { + return err + } + addr, err := addressArg(in.Address) + if err != nil { + return err + } + if in.KeyBlob == "" || len(in.KeyBlob) > 8<<10 { + return fail(400, "bad_key_blob") + } + if ok, err := s.Serves(addr); err != nil { + return err + } else if !ok { + return fail(421, "not_authoritative") + } + keyB64 := r.Header.Get("Stem-Key") + if !proto.PowOK(addr, keyB64, in.Pow.Nonce, s.cfg.Limits.PowBits) { + return fail(400, "bad_pow") + } + fp := proto.Fingerprint(pub) + l := s.cfg.Limits + for _, c := range []struct { + k string + n int + }{{"code:addr:" + addr, l.CodesPerAddressDay}, {"code:key:" + fp, l.CodesPerKeyDay}, {"code:ip:" + ipBucket(s.clientIP(r)), l.CodesPerIPDay}} { + if ok, err := s.hit(c.k, day, c.n); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + } + resp := map[string]string{"status": "code_sent"} + if sup, err := s.st.Suppressed(addr); err != nil { + return err + } else if sup { + writeJSON(w, 202, resp) + return nil + } + code, token := randomDigits(6), randomToken() + if err := s.st.PutPending(store.Pending{Address: addr, Fp: fp, AuthKey: keyB64, KeyBlob: in.KeyBlob, + CodeHash: codeHash(code, addr, fp), Token: token}); err != nil { + return err + } + unsub := randomToken() + if err := s.st.PutUnsubToken(unsub, addr); err != nil { + return err + } + holder, _, err := s.st.AddressAccount(addr) + if err != nil && !errors.Is(err, store.ErrNotFound) { + return err + } + link := s.cfg.URL + "/v1/verify-link?t=" + token + stop := s.cfg.URL + "/v1/unsubscribe?t=" + unsub + var subject, text string + if holder != nil && holder.Fp != fp { + subject = "A new device wants to use this address in Stem" + text = fmt.Sprintf("Someone asked to connect %s to a new Stem.\n\nIf that was you (for example on a new computer), your code is %s, or open:\n%s\n\nIf it wasn't you, ignore this email; nothing changes unless the code is used.\n\nNever email me again: %s\n", addr, code, link, stop) + } else { + subject = "Your Stem code: " + code + text = fmt.Sprintf("Your code to connect %s to Stem is %s.\n\nOr open: %s\n\nIf you didn't ask for this, ignore this email.\nThis wasn't me, never email me again: %s\n", addr, code, link, stop) + } + if err := s.mail.Send(mailer.Message{Kind: mailer.Verify, To: addr, Subject: subject, Body: text, Tag: "v-" + token[:12]}); err != nil { + log.Printf("verification mail to %s failed: %v", addr, err) + } + writeJSON(w, 202, resp) + return nil +} + +func (s *Server) verify(w http.ResponseWriter, r *http.Request, body []byte) error { + pub, err := s.signed(r, body) + if err != nil { + return err + } + var in struct{ Address, Code string } + if err := decode(body, &in); err != nil { + return err + } + addr, err := addressArg(in.Address) + if err != nil { + return err + } + fp := proto.Fingerprint(pub) + p, err := s.st.GetPending(addr, fp) + if errors.Is(err, store.ErrNotFound) { + return fail(400, "invalid_code") + } else if err != nil { + return err + } + // Spend the attempt before comparing, atomically, so parallel guesses + // can't each see attempts < 5. + if ok, err := s.st.TakePendingAttempt(addr, fp, 5); err != nil { + return err + } else if !ok { + return fail(400, "invalid_code") + } + if subtle.ConstantTimeCompare([]byte(codeHash(strings.TrimSpace(in.Code), addr, fp)), []byte(p.CodeHash)) != 1 { + return fail(400, "invalid_code") + } + if err := s.st.CompleteVerification(p); err != nil { + return err + } + writeJSON(w, 200, map[string]any{"cert": s.issueCert(addr, fp)}) + return nil +} + +// page renders a minimal HTML page. form, when set, adds a POST button +// carrying the token: links in emails only ever show a page, because mail +// scanners and link previews open every link they see. +func page(w http.ResponseWriter, status int, title, text string, form ...string) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'") + w.Header().Set("Referrer-Policy", "no-referrer") + w.WriteHeader(status) + fmt.Fprintf(w, `%s +

%s

%s

`, + html.EscapeString(title), html.EscapeString(title), html.EscapeString(text)) + if len(form) == 3 { + fmt.Fprintf(w, `
`, + html.EscapeString(form[0]), html.EscapeString(form[1]), html.EscapeString(form[2])) + } + fmt.Fprint(w, ``) +} + +func formToken(r *http.Request, body []byte) string { + v, err := url.ParseQuery(string(body)) + if err != nil { + return "" + } + return v.Get("t") +} + +func (s *Server) verifyLinkPage(w http.ResponseWriter, r *http.Request, _ []byte) error { + t := r.URL.Query().Get("t") + p, err := s.st.GetPendingByToken(t) + if errors.Is(err, store.ErrNotFound) { + page(w, 404, "Link expired", "This link was already used or has expired. Ask Stem for a new code.") + return nil + } else if err != nil { + return err + } + text := "Connect " + p.Address + " to the Stem that asked for it?" + if holder, _, err := s.st.AddressAccount(p.Address); err == nil && holder.Fp != p.Fp { + text = "This connects " + p.Address + " to a NEW device. The Stem using it now stops receiving its mail. Only continue if you asked for this." + } + page(w, 200, "Confirm your address", text, "/v1/verify-link", t, "Confirm") + return nil +} + +func (s *Server) verifyLink(w http.ResponseWriter, r *http.Request, body []byte) error { + p, err := s.st.GetPendingByToken(formToken(r, body)) + if errors.Is(err, store.ErrNotFound) { + page(w, 404, "Link expired", "This link was already used or has expired. Ask Stem for a new code.") + return nil + } else if err != nil { + return err + } + if err := s.st.CompleteVerification(p); err != nil { + return err + } + page(w, 200, "Address confirmed", "You can go back to Stem.") + return nil +} + +func (s *Server) cert(w http.ResponseWriter, r *http.Request, body []byte) error { + acc, addrs, err := s.own(r, body) + if err != nil { + return err + } + var in struct{ Address string } + if err := decode(body, &in); err != nil { + return err + } + addr := proto.NormalizeAddress(in.Address) + if !slices.Contains(addrs, addr) { + return fail(403, "not_your_address") + } + writeJSON(w, 200, map[string]any{"cert": s.issueCert(addr, acc.Fp)}) + return nil +} + +func (s *Server) revoke(w http.ResponseWriter, r *http.Request, body []byte) error { + acc, _, err := s.own(r, body) + if err != nil { + return err + } + if err := s.st.Revoke(acc.Fp); err != nil { + return err + } + writeJSON(w, 200, map[string]any{}) + return nil +} + +// ---- lookups ---- + +var errNotReachable = fail(404, "not_reachable") + +func (s *Server) keys(w http.ResponseWriter, r *http.Request, body []byte) error { + requester, _, err := s.certified(r, body) + if errors.Is(err, errOutsider) { + // Same answer as an unknown address: outsiders can't list who is here. + return errNotReachable + } else if err != nil { + return err + } + l := s.cfg.Limits + if ok, err := s.hit("lookup:min:"+requester, time.Minute, l.LookupBurst); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + if ok, err := s.hit("lookup:day:"+requester, day, l.LookupsPerDay); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + target, err := addressArg(r.URL.Query().Get("address")) + if err != nil { + return err + } + if ok, err := s.Serves(target); err != nil { + return err + } else if !ok { + return fail(421, "not_authoritative") + } + acc, _, err := s.st.AddressAccount(target) + if errors.Is(err, store.ErrNotFound) || err == nil && acc.Revoked { + return errNotReachable + } else if err != nil { + return err + } + if sus, err := s.st.Suspended(target); err != nil { + return err + } else if sus { + return errNotReachable + } + if acc.Unlisted { + if ok, err := s.st.Allows(acc.Fp, requester); err != nil { + return err + } else if !ok { + return errNotReachable + } + } + writeJSON(w, 200, map[string]string{"authKey": acc.AuthKey, "keyBlob": acc.KeyBlob}) + return nil +} + +func (s *Server) keystatus(w http.ResponseWriter, r *http.Request, _ []byte) error { + q := r.URL.Query() + addr := proto.NormalizeAddress(q.Get("address")) + acc, _, err := s.st.AddressAccount(addr) + current := err == nil && !acc.Revoked && acc.Fp == q.Get("fp") + if err != nil && !errors.Is(err, store.ErrNotFound) { + return err + } + writeJSON(w, 200, map[string]bool{"current": current}) + return nil +} + +// ---- lists and account ---- + +func (s *Server) listOp(op string) handler { + return func(w http.ResponseWriter, r *http.Request, body []byte) error { + acc, addrs, err := s.own(r, body) + if err != nil { + return err + } + var in struct{ Address string } + if err := decode(body, &in); err != nil { + return err + } + addr, err := addressArg(in.Address) + if err != nil { + return err + } + switch op { + case "allow": + err = s.st.Allow(acc.Fp, addr) + for _, own := range addrs { + if err == nil { + err = s.st.ClearKnock(addr, own) + } + } + case "unallow": + err = s.st.Unallow(acc.Fp, addr) + case "block": + if err = s.st.Block(acc.Fp, addr); err == nil { + err = s.maybeSuspend(addr) + } + case "unblock": + err = s.st.Unblock(acc.Fp, addr) + } + if err != nil { + return err + } + writeJSON(w, 200, map[string]any{}) + return nil + } +} + +func (s *Server) maybeSuspend(addr string) error { + l := s.cfg.Limits + if l.BlockSuspendCount <= 0 { + return nil + } + since := s.Now().Add(-time.Duration(l.BlockSuspendWindowHr) * time.Hour).UnixMilli() + n, err := s.st.DistinctBlockers(addr, since) + if err != nil || n < l.BlockSuspendCount { + return err + } + log.Printf("suspending %s: blocked by %d accounts", addr, n) + return s.st.Suspend(addr) +} + +func (s *Server) account(w http.ResponseWriter, r *http.Request, body []byte) error { + acc, _, err := s.own(r, body) + if err != nil { + return err + } + var in struct { + Unlisted *bool `json:"unlisted"` + } + if err := decode(body, &in); err != nil { + return err + } + if in.Unlisted != nil { + if err := s.st.SetUnlisted(acc.Fp, *in.Unlisted); err != nil { + return err + } + acc.Unlisted = *in.Unlisted + } + writeJSON(w, 200, map[string]any{"unlisted": acc.Unlisted}) + return nil +} + +func (s *Server) unsubscribePage(w http.ResponseWriter, r *http.Request, _ []byte) error { + t := r.URL.Query().Get("t") + addr, err := s.st.UnsubToken(t) + if errors.Is(err, store.ErrNotFound) { + page(w, 404, "Link not recognised", "This link is invalid.") + return nil + } else if err != nil { + return err + } + page(w, 200, "Stop all email from this relay?", "This relay will never email "+addr+" again, including codes to connect it to Stem, until it is connected from that address.", "/v1/unsubscribe", t, "Stop emailing me") + return nil +} + +func (s *Server) unsubscribe(w http.ResponseWriter, r *http.Request, body []byte) error { + addr, err := s.st.UnsubToken(formToken(r, body)) + if errors.Is(err, store.ErrNotFound) { + page(w, 404, "Link not recognised", "This link is invalid.") + return nil + } else if err != nil { + return err + } + if err := s.st.Suppress(addr); err != nil { + return err + } + page(w, 200, "You won't hear from us again", "This relay will not email "+addr+" again.") + return nil +} diff --git a/relay/internal/server/server_test.go b/relay/internal/server/server_test.go new file mode 100644 index 00000000..c3e66bf2 --- /dev/null +++ b/relay/internal/server/server_test.go @@ -0,0 +1,662 @@ +package server + +import ( + "bufio" + "bytes" + "context" + "crypto/ecdh" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/join3r/stem/relay/internal/authority" + "github.com/join3r/stem/relay/internal/config" + "github.com/join3r/stem/relay/internal/mailer" + "github.com/join3r/stem/relay/internal/proto" + "github.com/join3r/stem/relay/internal/store" +) + +// ---- harness: a global relay and a private relay for acme.test ---- + +type relay struct { + srv *Server + http *httptest.Server + mail *mailer.Log + st *store.Store +} + +type world struct { + t *testing.T + global *relay + acme *relay + static map[string]*authority.Record +} + +func newWorld(t *testing.T) *world { + w := &world{t: t, static: map[string]*authority.Record{}} + w.global = w.newRelay(true, nil) + w.acme = w.newRelay(false, []string{"acme.test"}) + // acme.test's DNS record points at the acme relay. + w.static["acme.test"] = &authority.Record{Relay: w.acme.http.URL, Keys: w.acme.srv.PublicKeys()} + return w +} + +func (w *world) newRelay(global bool, domains []string) *relay { + t := w.t + st, err := store.Open(filepath.Join(t.TempDir(), "relay.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + cfg := config.Defaults() + cfg.Global, cfg.Domains = global, domains + cfg.Limits.PowBits = 4 + cfg.Invite.Enabled = true + m := &mailer.Log{} + hs := httptest.NewUnstartedServer(nil) + hs.Start() + t.Cleanup(hs.Close) + cfg.URL = hs.URL + if !global { + cfg.GlobalURL = w.global.http.URL + } + res := authority.NewResolver(cfg.GlobalURL, nil) + res.Static = w.static + res.Lookup = func(ctx context.Context, name string) ([]string, error) { + return nil, nil // no records except Static + } + if !global { + res.GlobalKeys = w.global.srv.PublicKeys() + } + s, err := New(&cfg, st, m, res) + if err != nil { + t.Fatal(err) + } + hs.Config.Handler = s.Handler() + return &relay{srv: s, http: hs, mail: m, st: st} +} + +// ---- harness: a Stem client ---- + +type stem struct { + t *testing.T + home *relay + w *world + addr string + priv ed25519.PrivateKey + pub ed25519.PublicKey + enc *ecdh.PrivateKey + cert *proto.Cert + keyBlob string +} + +func (w *world) newStem(home *relay, addr string) *stem { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + enc, _ := ecdh.X25519().GenerateKey(rand.Reader) + encB64 := proto.B64(enc.PublicKey().Bytes()) + blob, _ := json.Marshal(map[string]any{"v": 1, "sig": proto.B64(pub), "enc": encB64, + "encSig": proto.B64(ed25519.Sign(priv, []byte("stem-key-v1\n"+encB64)))}) + return &stem{t: w.t, home: home, w: w, addr: addr, priv: priv, pub: pub, enc: enc, keyBlob: string(blob)} +} + +func (c *stem) do(base, method, path string, body any, withCert bool) (int, []byte) { + c.t.Helper() + var raw []byte + if body != nil { + raw, _ = json.Marshal(body) + } + req, _ := http.NewRequest(method, base+path, bytes.NewReader(raw)) + ts := time.Now().UnixMilli() + req.Header.Set("Stem-Key", proto.B64(c.pub)) + req.Header.Set("Stem-Time", strconv.FormatInt(ts, 10)) + req.Header.Set("Stem-Sig", proto.B64(ed25519.Sign(c.priv, []byte(proto.RequestSigningString(method, path, ts, raw))))) + if withCert { + cj, _ := json.Marshal(c.cert) + req.Header.Set("Stem-Cert", proto.B64(cj)) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + c.t.Fatal(err) + } + defer resp.Body.Close() + out, _ := io.ReadAll(resp.Body) + return resp.StatusCode, out +} + +var codeRe = regexp.MustCompile(`code (?:to connect \S+ to Stem )?is (\d{6})`) + +func lastCode(t *testing.T, m *mailer.Log, to string) string { + t.Helper() + msgs := m.Messages() + for i := len(msgs) - 1; i >= 0; i-- { + if msgs[i].To == to { + if mm := codeRe.FindStringSubmatch(msgs[i].Body); mm != nil { + return mm[1] + } + } + } + t.Fatalf("no code mailed to %s", to) + return "" +} + +func (c *stem) register() { + c.t.Helper() + nonce := proto.SolvePow(c.addr, proto.B64(c.pub), c.home.srv.cfg.Limits.PowBits) + st, body := c.do(c.home.http.URL, "POST", "/v1/register", map[string]any{"address": c.addr, "keyBlob": c.keyBlob, "pow": map[string]string{"nonce": nonce}}, false) + if st != 202 || !strings.Contains(string(body), "code_sent") { + c.t.Fatalf("register: %d %s", st, body) + } + code := lastCode(c.t, c.home.mail, c.addr) + st, body = c.do(c.home.http.URL, "POST", "/v1/verify", map[string]string{"address": c.addr, "code": code}, false) + if st != 200 { + c.t.Fatalf("verify: %d %s", st, body) + } + var out struct{ Cert *proto.Cert } + _ = json.Unmarshal(body, &out) + c.cert = out.Cert +} + +func (w *world) relayFor(addr string) *relay { + if proto.Domain(addr) == "acme.test" { + return w.acme + } + return w.global +} + +func (c *stem) lookup(addr string) (int, string, string) { + st, body := c.do(c.w.relayFor(addr).http.URL, "GET", "/v1/keys?address="+addr, nil, true) + var out struct{ AuthKey, KeyBlob string } + _ = json.Unmarshal(body, &out) + return st, out.AuthKey, out.KeyBlob +} + +func (c *stem) envelope(to string, encPub []byte, text string) *proto.Envelope { + c.t.Helper() + rk, err := ecdh.X25519().NewPublicKey(encPub) + if err != nil { + c.t.Fatal(err) + } + e := &proto.Envelope{V: 1, ID: proto.NewID(), Ts: time.Now().UnixMilli(), From: c.addr, To: to, Cert: c.cert} + sealed, err := proto.Seal(rk, []byte(text), e.ID, e.From, e.To, nil) + if err != nil { + c.t.Fatal(err) + } + e.Sealed = proto.B64(sealed) + if err := proto.SignEnvelope(e, c.priv); err != nil { + c.t.Fatal(err) + } + return e +} + +func (c *stem) send(to string, recipient *stem, text string) (*proto.Envelope, int, string) { + e := c.envelope(to, recipient.enc.PublicKey().Bytes(), text) + code, body := c.postEnv(e) + return e, code, body +} + +func (c *stem) postEnv(e *proto.Envelope) (int, string) { + raw, _ := json.Marshal(e) + resp, err := http.Post(c.w.relayFor(e.To).http.URL+"/v1/mail", "application/json", bytes.NewReader(raw)) + if err != nil { + c.t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(b) +} + +// held returns envelopes waiting on the stream (reads until ": ready"). +func (c *stem) held() []proto.Envelope { + c.t.Helper() + path := "/v1/stream" + ts := time.Now().UnixMilli() + req, _ := http.NewRequest("GET", c.home.http.URL+path, nil) + req.Header.Set("Stem-Key", proto.B64(c.pub)) + req.Header.Set("Stem-Time", strconv.FormatInt(ts, 10)) + req.Header.Set("Stem-Sig", proto.B64(ed25519.Sign(c.priv, []byte(proto.RequestSigningString("GET", path, ts, nil))))) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + c.t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + b, _ := io.ReadAll(resp.Body) + c.t.Fatalf("stream: %d %s", resp.StatusCode, b) + } + var out []proto.Envelope + sc := bufio.NewScanner(resp.Body) + sc.Buffer(make([]byte, 1<<20), 64<<20) + for sc.Scan() { + line := sc.Text() + if line == ": ready" { + break + } + if d, ok := strings.CutPrefix(line, "data: "); ok { + var e proto.Envelope + if err := json.Unmarshal([]byte(d), &e); err != nil { + c.t.Fatal(err) + } + out = append(out, e) + } + } + return out +} + +func (c *stem) ack(ids ...string) { + if st, b := c.do(c.home.http.URL, "POST", "/v1/ack", map[string]any{"ids": ids}, false); st != 200 { + c.t.Fatalf("ack: %d %s", st, b) + } +} + +func (c *stem) list(op, addr string) { + if st, b := c.do(c.home.http.URL, "POST", "/v1/"+op, map[string]string{"address": addr}, false); st != 200 { + c.t.Fatalf("%s: %d %s", op, st, b) + } +} + +func (c *stem) status(to, id string) string { + _, b := c.do(c.w.relayFor(to).http.URL, "GET", "/v1/mail/status?id="+id, nil, true) + var out struct{ Status string } + _ = json.Unmarshal(b, &out) + return out.Status +} + +// ---- tests ---- + +func TestCrossRelayDelivery(t *testing.T) { + w := newWorld(t) + vlado := w.newStem(w.global, "vlado@gmail.test") + martin := w.newStem(w.acme, "martin@acme.test") + vlado.register() + martin.register() + + st, authKey, blob := vlado.lookup("martin@acme.test") + if st != 200 || authKey != proto.B64(martin.pub) || blob != martin.keyBlob { + t.Fatalf("lookup: %d %q", st, authKey) + } + + env, code, _ := vlado.send("martin@acme.test", martin, "hello martin") + if code != 202 { + t.Fatalf("send: %d", code) + } + if s := vlado.status("martin@acme.test", env.ID); s != "sent" { + t.Fatalf("stranger status = %q, want sent", s) + } + got := martin.held() + if len(got) != 1 { + t.Fatalf("held = %d", len(got)) + } + sealed, _ := proto.UnB64(got[0].Sealed) + plain, err := proto.Open(martin.enc, sealed, got[0].ID, got[0].From, got[0].To) + if err != nil || string(plain) != "hello martin" { + t.Fatalf("open: %v %q", err, plain) + } + martin.ack(got[0].ID) + if len(martin.held()) != 0 { + t.Fatal("acked mail still held") + } + // Once martin accepts vlado, vlado sees real status. + martin.list("allow", "vlado@gmail.test") + if s := vlado.status("martin@acme.test", env.ID); s != "delivered" { + t.Fatalf("contact status = %q, want delivered", s) + } + // Replay is refused. + if c, _ := vlado.postEnv(env); c != 409 { + t.Fatalf("replay: %d", c) + } +} + +func TestForgedSenderRejected(t *testing.T) { + w := newWorld(t) + martin := w.newStem(w.acme, "martin@acme.test") + martin.register() + mallory := w.newStem(w.global, "mallory@gmail.test") + mallory.register() + + // The global relay is not the authority for acme.test, so a cert it + // signs for an acme address must not verify anywhere. + forged := w.global.srv.issueCert("ceo@acme.test", proto.Fingerprint(mallory.pub)) + mallory.cert = forged + mallory.addr = "ceo@acme.test" + _, code, _ := mallory.send("martin@acme.test", martin, "wire the money") + if code != 400 { + t.Fatalf("forged cert accepted: %d", code) + } + if len(martin.held()) != 0 { + t.Fatal("forged mail held") + } + + // A valid cert but an envelope signed by a different key. + mallory2 := w.newStem(w.global, "mallory2@gmail.test") + mallory2.register() + e := mallory2.envelope("martin@acme.test", martin.enc.PublicKey().Bytes(), "x") + other, otherPriv, _ := ed25519.GenerateKey(rand.Reader) + _ = other + _ = proto.SignEnvelope(e, otherPriv) + if c, _ := mallory2.postEnv(e); c != 400 { + t.Fatalf("mismatched signer accepted: %d", c) + } +} + +func TestOneKnockAndBlock(t *testing.T) { + w := newWorld(t) + a := w.newStem(w.global, "a@gmail.test") + b := w.newStem(w.global, "b@gmail.test") + a.register() + b.register() + + for i := range 3 { + if _, c, _ := a.send(b.addr, b, "knock "+strconv.Itoa(i)); c != 202 { + t.Fatalf("send %d: %d", i, c) + } + } + if n := len(b.held()); n != 1 { + t.Fatalf("stranger got %d mails held, want 1 (one knock)", n) + } + b.list("allow", a.addr) + a.send(b.addr, b, "after accept") + if n := len(b.held()); n != 2 { + t.Fatalf("after accept held = %d, want 2", n) + } + b.list("block", a.addr) + a.send(b.addr, b, "after block") + if n := len(b.held()); n != 2 { + t.Fatalf("after block held = %d, want 2", n) + } +} + +func TestUnlistedLooksUnregistered(t *testing.T) { + w := newWorld(t) + a := w.newStem(w.global, "a@gmail.test") + b := w.newStem(w.global, "b@gmail.test") + a.register() + b.register() + + if st, _ := b.do(w.global.http.URL, "PUT", "/v1/account", map[string]bool{"unlisted": true}, false); st != 200 { + t.Fatal(st) + } + st1, _, _ := a.lookup(b.addr) + st2, _, _ := a.lookup("nobody@gmail.test") + if st1 != 404 || st2 != 404 { + t.Fatalf("unlisted %d, unregistered %d; want both 404", st1, st2) + } + b.list("allow", a.addr) + if st, _, _ := a.lookup(b.addr); st != 200 { + t.Fatalf("allowed lookup: %d", st) + } +} + +func TestUnregisteredRecipientDropped(t *testing.T) { + w := newWorld(t) + a := w.newStem(w.global, "a@gmail.test") + a.register() + ghost := w.newStem(w.global, "ghost@gmail.test") + if _, c, body := a.send(ghost.addr, ghost, "hi"); c != 202 || !strings.Contains(body, "sent") { + t.Fatalf("send to unregistered: %d %s", c, body) + } + if n, _, _ := w.global.st.QueueUsage(ghost.addr); n != 0 { + t.Fatal("mail to unregistered was stored") + } +} + +func TestReRegisterWarnsOwner(t *testing.T) { + w := newWorld(t) + a := w.newStem(w.global, "a@gmail.test") + a.register() + laptop2 := w.newStem(w.global, "a@gmail.test") + laptop2.register() // same flow, same answers + msgs := w.global.mail.Messages() + if !strings.Contains(msgs[len(msgs)-1].Subject, "new device") { + t.Fatalf("second registration subject = %q", msgs[len(msgs)-1].Subject) + } + // The old key no longer gets certificates for the address. + if st, _ := a.do(w.global.http.URL, "POST", "/v1/cert", map[string]string{"address": a.addr}, false); st != 403 { + t.Fatalf("old key cert: %d", st) + } + var ks struct{ Current bool } + resp, _ := http.Get(w.global.http.URL + "/v1/keystatus?address=" + a.addr + "&fp=" + proto.Fingerprint(a.pub)) + _ = json.NewDecoder(resp.Body).Decode(&ks) + resp.Body.Close() + if ks.Current { + t.Fatal("old key still current") + } +} + +func TestInvites(t *testing.T) { + w := newWorld(t) + g := w.global + g.srv.cfg.Invite.GlobalPerDay = 2 + senders := []*stem{w.newStem(g, "s1@gmail.test"), w.newStem(g, "s2@gmail.test"), w.newStem(g, "s3@gmail.test")} + for _, s := range senders { + s.register() + // Not new: backdate verification so new-address limits don't apply. + _, _ = g.st.DB().Exec(`UPDATE addresses SET verified = 0 WHERE address = ?`, s.addr) + } + invite := func(s *stem, to string) { + if st, b := s.do(g.http.URL, "POST", "/v1/invite", map[string]string{"address": to}, true); st != 202 { + t.Fatalf("invite: %d %s", st, b) + } + } + invite(senders[0], "friend@gmail.test") + invite(senders[1], "friend@gmail.test") + invite(senders[0], "other@gmail.test") + invite(senders[2], "third@gmail.test") + invite(senders[0], "s2@gmail.test") // registered: no invite + before := len(g.mail.Messages()) + if err := g.srv.DispatchInvites(); err != nil { + t.Fatal(err) + } + sentNow := g.mail.Messages()[before:] + if len(sentNow) != 2 { + t.Fatalf("global budget: sent %d, want 2", len(sentNow)) + } + if sentNow[0].To != "friend@gmail.test" || !strings.Contains(sentNow[0].Subject, "and 1 other") { + t.Fatalf("batched invite: %+v", sentNow[0]) + } + for _, m := range sentNow { + if m.To == "s2@gmail.test" { + t.Fatal("invited a registered address") + } + } + // Next run: budget spent for today. + if err := g.srv.DispatchInvites(); err != nil { + t.Fatal(err) + } + if len(g.mail.Messages()) != before+2 { + t.Fatal("sent past the global budget") + } +} + +func TestBreakerPausesInvites(t *testing.T) { + w := newWorld(t) + g := w.global + g.srv.cfg.Invite.BreakerMinSample = 2 + for i := range 4 { + tag := "i-t" + strconv.Itoa(i) + _ = g.st.LogInvite("x"+strconv.Itoa(i)+"@gmail.test", tag, []string{"s@gmail.test"}) + if i == 0 { + _ = g.st.LogMailEvent(tag, "complaint") + } + } + open, _, err := g.srv.breakerOpen() + if err != nil || !open { + t.Fatalf("breaker open = %v, %v", open, err) + } +} + +func TestWellKnownAndInfo(t *testing.T) { + w := newWorld(t) + resp, err := http.Get(w.acme.http.URL + "/.well-known/stem-relay") + if err != nil { + t.Fatal(err) + } + var wk struct { + URL string + Keys []struct{ Kid, Pub string } + } + _ = json.NewDecoder(resp.Body).Decode(&wk) + resp.Body.Close() + if wk.URL != w.acme.http.URL || len(wk.Keys) != 1 { + t.Fatalf("well-known: %+v", wk) + } +} + +func TestEmailLinksNeedAPost(t *testing.T) { + w := newWorld(t) + g := w.global + owner := w.newStem(g, "o@gmail.test") + owner.register() + thief := w.newStem(g, "o@gmail.test") + nonce := proto.SolvePow(thief.addr, proto.B64(thief.pub), g.srv.cfg.Limits.PowBits) + thief.do(g.http.URL, "POST", "/v1/register", map[string]any{"address": thief.addr, "keyBlob": thief.keyBlob, "pow": map[string]string{"nonce": nonce}}, false) + msgs := g.mail.Messages() + link := regexp.MustCompile(`https?://\S+/v1/verify-link\?t=\S+`).FindString(msgs[len(msgs)-1].Body) + unsub := regexp.MustCompile(`https?://\S+/v1/unsubscribe\?t=\S+`).FindString(msgs[len(msgs)-1].Body) + // A mail scanner opening both links changes nothing. + for _, l := range []string{link, unsub} { + resp, err := http.Get(l) + if err != nil || resp.StatusCode != 200 { + t.Fatalf("GET %s: %v %v", l, err, resp) + } + resp.Body.Close() + } + if acc, _, _ := g.st.AddressAccount(owner.addr); acc.Fp != proto.Fingerprint(owner.pub) { + t.Fatal("GET on the verify link moved the address") + } + if sup, _ := g.st.Suppressed(owner.addr); sup { + t.Fatal("GET on the unsubscribe link suppressed the address") + } + // Pressing the button does. + tok := link[strings.Index(link, "t=")+2:] + resp, _ := http.PostForm(g.http.URL+"/v1/verify-link", map[string][]string{"t": {tok}}) + resp.Body.Close() + if acc, _, _ := g.st.AddressAccount(owner.addr); acc.Fp != proto.Fingerprint(thief.pub) { + t.Fatal("POST on the verify link did not complete") + } +} + +func TestVerifyAttemptsAreCapped(t *testing.T) { + w := newWorld(t) + g := w.global + c := w.newStem(g, "c@gmail.test") + nonce := proto.SolvePow(c.addr, proto.B64(c.pub), g.srv.cfg.Limits.PowBits) + c.do(g.http.URL, "POST", "/v1/register", map[string]any{"address": c.addr, "keyBlob": c.keyBlob, "pow": map[string]string{"nonce": nonce}}, false) + code := lastCode(t, g.mail, c.addr) + done := make(chan int, 20) + for i := range 20 { + go func() { + st, _ := c.do(g.http.URL, "POST", "/v1/verify", map[string]string{"address": c.addr, "code": strconv.Itoa(100000 + i)}, false) + done <- st + }() + } + for range 20 { + <-done + } + if st, _ := c.do(g.http.URL, "POST", "/v1/verify", map[string]string{"address": c.addr, "code": code}, false); st != 400 { + t.Fatalf("right code after 20 wrong guesses: %d, want 400", st) + } +} + +func TestStreamFollowsTheKeyNotTheAddress(t *testing.T) { + w := newWorld(t) + g := w.global + old := w.newStem(g, "k@gmail.test") + old.register() + sub := g.srv.hub.subscribe(proto.Fingerprint(old.pub)) + defer g.srv.hub.unsubscribe(sub) + fresh := w.newStem(g, "k@gmail.test") + fresh.register() + sender := w.newStem(g, "s@gmail.test") + sender.register() + sender.send(fresh.addr, fresh, "for the new key") + select { + case <-sub: + t.Fatal("the old key's stream received mail for a moved address") + case <-time.After(100 * time.Millisecond): + } + if n := len(fresh.held()); n != 1 { + t.Fatalf("new key held %d", n) + } +} + +func TestClientIPIgnoresForgedForwardedFor(t *testing.T) { + s := &Server{cfg: &config.Config{TrustProxy: true}} + r := httptest.NewRequest("GET", "/", nil) + r.RemoteAddr = "172.18.0.2:5000" + r.Header.Set("X-Forwarded-For", "9.9.9.9, 203.0.113.7") + if got := s.clientIP(r); got != "203.0.113.7" { + t.Fatalf("clientIP = %q, want the proxy-appended 203.0.113.7", got) + } + s.cfg.TrustProxy = false + if got := s.clientIP(r); got != "172.18.0.2" { + t.Fatalf("clientIP without trust_proxy = %q", got) + } +} + +func TestIPBucketGroupsIPv6By64(t *testing.T) { + if a, b := ipBucket("2001:db8:1:2::1"), ipBucket("2001:db8:1:2:ffff::9"); a != b || a != "2001:db8:1:2::/64" { + t.Fatalf("same /64 split: %q %q", a, b) + } + if ipBucket("2001:db8:1:3::1") == ipBucket("2001:db8:1:2::1") { + t.Fatal("different /64s merged") + } + if got := ipBucket("203.0.113.7"); got != "203.0.113.7" { + t.Fatalf("ipv4 bucket %q", got) + } + if got := ipBucket("::ffff:203.0.113.7"); got != "::ffff:203.0.113.7" { + t.Fatalf("mapped ipv4 bucket %q", got) + } +} + +func TestClosedRelay(t *testing.T) { + w := newWorld(t) + w.acme.srv.cfg.Closed = true + w.static["acme.test"].Closed = true + vlado := w.newStem(w.global, "vlado@gmail.test") + martin := w.newStem(w.acme, "martin@acme.test") + anna := w.newStem(w.acme, "anna@acme.test") + vlado.register() + martin.register() + anna.register() + + // Outsiders can't see who is there, and can't deliver. + if st, _, _ := vlado.lookup("martin@acme.test"); st != 404 { + t.Fatalf("outsider lookup: %d, want 404", st) + } + if _, code, _ := vlado.send("martin@acme.test", martin, "hi"); code != 403 { + t.Fatalf("outsider mail: %d, want 403", code) + } + // The record's access=closed makes the global relay refuse acme senders. + if st, _, _ := martin.lookup("vlado@gmail.test"); st != 404 { + t.Fatalf("closed-domain lookup on the global relay: %d, want 404", st) + } + if _, code, _ := martin.send("vlado@gmail.test", vlado, "out"); code != 403 { + t.Fatalf("closed-domain mail on the global relay: %d, want 403", code) + } + if len(martin.held()) != 0 || len(vlado.held()) != 0 { + t.Fatal("mail crossed a closed boundary") + } + // Colleagues still reach each other. + if st, _, _ := anna.lookup("martin@acme.test"); st != 200 { + t.Fatalf("colleague lookup: %d", st) + } + if _, code, _ := anna.send("martin@acme.test", martin, "lunch?"); code != 202 { + t.Fatalf("colleague mail: %d", code) + } + // A partner domain is let in. + w.acme.srv.cfg.PartnerDomains = []string{"gmail.test"} + if _, code, _ := vlado.send("martin@acme.test", martin, "partner"); code != 202 { + t.Fatalf("partner mail: %d", code) + } +} diff --git a/relay/internal/server/share.go b/relay/internal/server/share.go new file mode 100644 index 00000000..4c0ecb1d --- /dev/null +++ b/relay/internal/server/share.go @@ -0,0 +1,378 @@ +package server + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "html" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/join3r/stem/relay/internal/store" +) + +// Public share links (PROTOCOL.md, "Share links"). A Stem encrypts the +// snapshot itself and keeps the key in the link's fragment, so a blob here +// is opaque bytes: the relay can count, expire and remove them, never read +// them. Uploads need a sender certificate, so every share traces back to a +// verified address. + +var shareIDRe = regexp.MustCompile(`^[A-Za-z0-9_-]{22}$`) + +// A blob is version byte 1, a 12-byte nonce, then AES-GCM output (at least +// the 16-byte tag). +const shareMinBytes = 1 + 12 + 16 + +func newShareID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return base64.RawURLEncoding.EncodeToString(b) +} + +func (s *Server) shareOn() error { + if !s.cfg.Share.Enabled { + return fail(404, "share_disabled") + } + return nil +} + +func (s *Server) shareURL(id string) string { return s.cfg.Share.ViewerURL + "/s/" + id } + +func (s *Server) postShare(w http.ResponseWriter, r *http.Request, body []byte) error { + if err := s.shareOn(); err != nil { + return err + } + addr, fp, err := s.certified(r, body) + if err != nil { + return err + } + if len(body) < shareMinBytes || body[0] != 1 { + return fail(400, "bad_blob") + } + cfg := s.cfg.Share + // No ttl means the longest allowed; with max_ttl_days = 0 that is never. + days := cfg.MaxTTLDays + if v := r.URL.Query().Get("ttl"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 3650 || cfg.MaxTTLDays > 0 && n > cfg.MaxTTLDays { + return fail(400, "bad_ttl") + } + days = n + } + var expires int64 + if days > 0 { + expires = s.Now().Add(time.Duration(days) * day).UnixMilli() + } + cardTitle, cardDesc, err := parseCard(r.URL.Query().Get("card")) + if err != nil { + return err + } + live, used, err := s.st.OwnerShareUsage(addr) + if err != nil { + return err + } + if cfg.MaxLive > 0 && live >= cfg.MaxLive { + return fail(507, "too_many_links") + } + if cfg.QuotaBytes > 0 && used+int64(len(body)) > cfg.QuotaBytes { + return fail(507, "quota_full") + } + if ok, err := s.hit("share:addr:"+addr, day, cfg.PerAddressDay); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + sh := &store.Share{ID: newShareID(), Owner: addr, Fp: fp, Size: int64(len(body)), Blob: body, + Created: s.Now().UnixMilli(), Expires: expires, CardTitle: cardTitle, CardDesc: cardDesc} + if err := s.st.PutShare(sh); err != nil { + return err + } + writeJSON(w, 200, map[string]any{"id": sh.ID, "url": s.shareURL(sh.ID), "expires": expires}) + return nil +} + +func (s *Server) getShare(w http.ResponseWriter, r *http.Request, _ []byte) error { + if err := s.shareOn(); err != nil { + return err + } + id := r.PathValue("id") + if !shareIDRe.MatchString(id) { + return fail(404, "not_found") + } + sh, err := s.st.GetShare(id) + if errors.Is(err, store.ErrNotFound) { + return fail(404, "not_found") + } else if err != nil { + return err + } + h := w.Header() + h.Set("X-Robots-Tag", "noindex") + h.Set("X-Content-Type-Options", "nosniff") + if sh.Removed > 0 { + // The owner revoked it or it was taken down: say which, nothing else. + why := "revoked" + if sh.RemovedWhy != "owner" { + why = "taken_down" + } + h.Set("Cache-Control", "no-store") + writeJSON(w, 410, map[string]string{"error": "gone", "why": why}) + return nil + } + // Short cache, so a revoke takes effect within a minute. + h.Set("Cache-Control", "private, max-age=60") + h.Set("Content-Type", "application/octet-stream") + h.Set("Content-Length", strconv.Itoa(len(sh.Blob))) + _, _ = w.Write(sh.Blob) + return nil +} + +func (s *Server) deleteShare(w http.ResponseWriter, r *http.Request, body []byte) error { + if err := s.shareOn(); err != nil { + return err + } + addr, _, err := s.certified(r, body) + if err != nil { + return err + } + id := r.PathValue("id") + sh, err := s.st.GetShare(id) + if errors.Is(err, store.ErrNotFound) { + writeJSON(w, 200, map[string]any{}) // already gone + return nil + } else if err != nil { + return err + } + // The address owns it, not the key: a new device holding the same + // address can still revoke what an old one shared. + if sh.Owner != addr { + return fail(404, "not_found") + } + if _, err := s.st.RemoveShare(id, "owner"); err != nil { + return err + } + writeJSON(w, 200, map[string]any{}) + return nil +} + +func (s *Server) listShares(w http.ResponseWriter, r *http.Request, body []byte) error { + if err := s.shareOn(); err != nil { + return err + } + addr, _, err := s.certified(r, body) + if err != nil { + return err + } + rows, err := s.st.OwnerShares(addr) + if err != nil { + return err + } + out := make([]map[string]any, 0, len(rows)) + for _, sh := range rows { + out = append(out, map[string]any{"id": sh.ID, "url": s.shareURL(sh.ID), "size": sh.Size, "created": sh.Created, "expires": sh.Expires}) + } + writeJSON(w, 200, map[string]any{"shares": out}) + return nil +} + +var shareReasons = map[string]bool{"illegal": true, "privacy": true, "harassment": true, "spam": true, "other": true} + +// printable drops control characters (C0 except newline, DEL, C1): report +// text is anonymous and ends up in an operator's terminal (share-reports). +func printable(v string) string { + return strings.Map(func(r rune) rune { + if r == '\n' || r >= 0x20 && r != 0x7f && (r < 0x80 || r > 0x9f) { + return r + } + return -1 + }, v) +} + +var shareKeyRe = regexp.MustCompile(`^[A-Za-z0-9_-]{43}$`) + +func clip(v string, n int) string { + v = strings.TrimSpace(printable(v)) + if len(v) <= n { + return v + } + v = v[:n] + for !utf8.ValidString(v) { + v = v[:len(v)-1] + } + return v +} + +// reportShare is the notice-and-action form. A reporter may include the +// link's key, consenting to let the operator see what they saw. +func (s *Server) reportShare(w http.ResponseWriter, r *http.Request, body []byte) error { + if err := s.shareOn(); err != nil { + return err + } + id := r.PathValue("id") + if !shareIDRe.MatchString(id) { + return fail(404, "not_found") + } + var in struct { + Reason string `json:"reason"` + Detail string `json:"detail"` + Contact string `json:"contact"` + Key string `json:"key"` + } + if err := decode(body, &in); err != nil { + return err + } + if !shareReasons[in.Reason] { + return fail(400, "bad_reason") + } + ip := ipBucket(s.clientIP(r)) + if ok, err := s.hit("share:report:"+ip, time.Hour, s.cfg.Share.ReportsPerIPHour); err != nil { + return err + } else if !ok { + return fail(429, "rate_limited") + } + if _, err := s.st.GetShare(id); errors.Is(err, store.ErrNotFound) { + return fail(404, "not_found") + } else if err != nil { + return err + } + if in.Key != "" && !shareKeyRe.MatchString(in.Key) { + return fail(400, "bad_key") + } + rep := &store.ShareReport{Share: id, Reason: in.Reason, Detail: clip(in.Detail, 4000), + Contact: strings.ReplaceAll(clip(in.Contact, 256), "\n", " "), Key: in.Key, IP: ip} + if err := s.st.AddShareReport(rep); err != nil { + return err + } + log.Printf("share: report on %s (%s)", id, in.Reason) + writeJSON(w, 202, map[string]any{}) + return nil +} + +// ---- link previews ---- + +const ( + cardTitleMax = 120 + cardDescMax = 300 +) + +// parseCard reads the optional link preview a Stem publishes in the clear +// (?card=base64url(JSON {"t","d"})); it rides in the signed path. One line +// each, printable only, clipped. +func parseCard(v string) (string, string, error) { + if v == "" { + return "", "", nil + } + raw, err := base64.RawURLEncoding.DecodeString(v) + if err != nil || len(raw) > 4096 { + return "", "", fail(400, "bad_card") + } + var c struct { + T string `json:"t"` + D string `json:"d"` + } + if json.Unmarshal(raw, &c) != nil { + return "", "", fail(400, "bad_card") + } + line := func(v string, n int) string { return clip(strings.Join(strings.Fields(printable(v)), " "), n) } + return line(c.T, cardTitleMax), line(c.D, cardDescMax), nil +} + +// sharePage serves the viewer's index.html for /s/ with that share's +// preview tags (between markers), so chat apps that unfurl +// the link show its card. The page and every response say noindex: previews +// yes, search results no. The fragment (the key) never reaches here. +func (s *Server) sharePage(w http.ResponseWriter, r *http.Request, _ []byte) error { + if !s.onViewerHost(r) { + return fail(404, "not_found") + } + page, err := os.ReadFile(filepath.Join(s.cfg.Share.ViewerDir, "index.html")) + if err != nil { + return err + } + id := r.PathValue("id") + var sh *store.Share + if shareIDRe.MatchString(id) { + if got, err := s.st.GetShare(id); err == nil { + sh = got + } else if !errors.Is(err, store.ErrNotFound) { + return err + } + } + h := w.Header() + h.Set("X-Robots-Tag", "noindex, nofollow, noarchive") + h.Set("Cache-Control", "no-cache") + h.Set("Content-Type", "text/html; charset=utf-8") + if sh == nil || sh.Removed > 0 { + page = withCard(page, "Link removed · Stem", "This shared page is no longer available.", s.cfg.Share.ViewerURL, "") + } else if sh.CardTitle != "" || sh.CardDesc != "" { + title := sh.CardTitle + if title == "" { + title = "Shared from Stem" + } + page = withCard(page, title, sh.CardDesc, s.cfg.Share.ViewerURL, s.shareURL(sh.ID)) + } + _, _ = w.Write(page) + return nil +} + +// onViewerHost: the viewer is only served on its own origin (viewer_url), +// never on the relay's, so shared pages don't run where relay calls do. +func (s *Server) onViewerHost(r *http.Request) bool { + if !s.cfg.Share.Enabled || s.cfg.Share.ViewerDir == "" { + return false + } + u, err := url.Parse(s.cfg.Share.ViewerURL) + return err == nil && strings.EqualFold(r.Host, u.Host) +} + +// viewerAsset serves the viewer's built files (hashed assets, icons). +func (s *Server) viewerAsset(w http.ResponseWriter, r *http.Request) { + if !s.onViewerHost(r) { + http.NotFound(w, r) + return + } + h := w.Header() + h.Set("X-Robots-Tag", "noindex, nofollow, noarchive") + h.Set("X-Content-Type-Options", "nosniff") + if strings.HasPrefix(r.URL.Path, "/assets/") { + h.Set("Cache-Control", "public, max-age=31536000, immutable") // names carry a content hash + } else { + h.Set("Cache-Control", "public, max-age=3600") + } + http.FileServer(http.Dir(s.cfg.Share.ViewerDir)).ServeHTTP(w, r) +} + +func withCard(page []byte, title, desc, base, url string) []byte { + start, end := []byte(""), []byte("") + i, j := strings.Index(string(page), string(start)), strings.Index(string(page), string(end)) + if i < 0 || j < i { + return page + } + e := html.EscapeString + var b strings.Builder + b.WriteString("\n") + b.WriteString(" " + e(title) + "\n") + b.WriteString(` ` + "\n") + b.WriteString(` ` + "\n") + b.WriteString(` ` + "\n") + if desc != "" { + b.WriteString(` ` + "\n") + b.WriteString(` ` + "\n") + } + if url != "" { + b.WriteString(` ` + "\n") + } + b.WriteString(` ` + "\n") + b.WriteString(` ` + "\n ") + out := append([]byte{}, page[:i]...) + out = append(out, b.String()...) + return append(out, page[j:]...) +} diff --git a/relay/internal/server/share_test.go b/relay/internal/server/share_test.go new file mode 100644 index 00000000..9e53e392 --- /dev/null +++ b/relay/internal/server/share_test.go @@ -0,0 +1,310 @@ +package server + +import ( + "bytes" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/join3r/stem/relay/internal/proto" +) + +// doRaw is do with an opaque body (share blobs aren't JSON). +func (c *stem) doRaw(base, method, path string, raw []byte, withCert bool) (int, []byte) { + c.t.Helper() + req, _ := http.NewRequest(method, base+path, bytes.NewReader(raw)) + ts := time.Now().UnixMilli() + req.Header.Set("Stem-Key", proto.B64(c.pub)) + req.Header.Set("Stem-Time", strconv.FormatInt(ts, 10)) + req.Header.Set("Stem-Sig", proto.B64(ed25519.Sign(c.priv, []byte(proto.RequestSigningString(method, path, ts, raw))))) + if withCert { + cj, _ := json.Marshal(c.cert) + req.Header.Set("Stem-Cert", proto.B64(cj)) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + c.t.Fatal(err) + } + defer resp.Body.Close() + out, _ := io.ReadAll(resp.Body) + return resp.StatusCode, out +} + +func get(t *testing.T, url string) (int, []byte) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + out, _ := io.ReadAll(resp.Body) + return resp.StatusCode, out +} + +func blob(n int) []byte { + b := make([]byte, n) + b[0] = 1 + return b +} + +func TestShareLinks(t *testing.T) { + w := newWorld(t) + g := w.global + a := w.newStem(g, "a@gmail.test") + a.register() + b := w.newStem(w.acme, "b@acme.test") + b.register() + + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares", blob(64), true); code != 404 { + t.Fatalf("disabled share answered %d", code) + } + g.srv.cfg.Share.Enabled = true + g.srv.cfg.Share.ViewerURL = "https://share.example" + + // A certificate is required, and the blob must look like one. + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares", blob(64), false); code != 401 { + t.Fatalf("uncertified upload answered %d", code) + } + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares", []byte(strings.Repeat("x", 64)), true); code != 400 { + t.Fatalf("bad blob answered %d", code) + } + + code, out := a.doRaw(g.http.URL, "POST", "/v1/shares", blob(64), true) + if code != 200 { + t.Fatalf("upload: %d %s", code, out) + } + var made struct { + ID string `json:"id"` + URL string `json:"url"` + Expires int64 `json:"expires"` + } + _ = json.Unmarshal(out, &made) + // No ttl: the longest allowed (30 days by default). + if !shareIDRe.MatchString(made.ID) || made.URL != "https://share.example/s/"+made.ID || + made.Expires < time.Now().Add(29*day).UnixMilli() || made.Expires > time.Now().Add(31*day).UnixMilli() { + t.Fatalf("upload answered %s", out) + } + + // Anyone can fetch the bytes. + code, got := get(t, g.http.URL+"/v1/shares/"+made.ID) + if code != 200 || !bytes.Equal(got, blob(64)) { + t.Fatalf("fetch: %d (%d bytes)", code, len(got)) + } + if code, _ := get(t, g.http.URL+"/v1/shares/AAAAAAAAAAAAAAAAAAAAAA"); code != 404 { + t.Fatalf("unknown id answered %d", code) + } + + // An address from another relay can share too, and lists only its own. + code, out = b.doRaw(g.http.URL, "POST", "/v1/shares?ttl=7", blob(40), true) + if code != 200 { + t.Fatalf("cross-relay upload: %d %s", code, out) + } + var bMade struct { + ID string `json:"id"` + Expires int64 `json:"expires"` + } + _ = json.Unmarshal(out, &bMade) + if bMade.Expires < time.Now().Add(6*day).UnixMilli() { + t.Fatalf("ttl ignored: %s", out) + } + _, out = a.do(g.http.URL, "GET", "/v1/shares", nil, true) + if !strings.Contains(string(out), made.ID) || strings.Contains(string(out), bMade.ID) { + t.Fatalf("list: %s", out) + } + + // Only the owner revokes; then the link says gone. + if code, _ := b.do(g.http.URL, "DELETE", "/v1/shares/"+made.ID, nil, true); code != 404 { + t.Fatalf("stranger's revoke answered %d", code) + } + if code, _ := a.do(g.http.URL, "DELETE", "/v1/shares/"+made.ID, nil, true); code != 200 { + t.Fatalf("revoke answered %d", code) + } + code, out = get(t, g.http.URL+"/v1/shares/"+made.ID) + if code != 410 || !strings.Contains(string(out), "revoked") { + t.Fatalf("revoked fetch: %d %s", code, out) + } + + // Reports land in the store; a bad reason doesn't. + post := func(id string, body string) int { + resp, err := http.Post(g.http.URL+"/v1/shares/"+id+"/report", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + return resp.StatusCode + } + if c := post(bMade.ID, `{"reason":"nope"}`); c != 400 { + t.Fatalf("bad reason answered %d", c) + } + if c := post(bMade.ID, `{"reason":"illegal","key":"not a key"}`); c != 400 { + t.Fatalf("bad key answered %d", c) + } + key := strings.Repeat("k", 43) + if c := post(bMade.ID, `{"reason":"illegal","detail":"bad\u001b[2Jstuff\nline two","contact":"a\u009b@b\nc","key":"`+key+`"}`); c != 202 { + t.Fatalf("report answered %d", c) + } + reps, _ := g.st.OpenShareReports() + if len(reps) != 1 || reps[0].Share != bMade.ID || reps[0].Key != key { + t.Fatalf("reports: %+v", reps) + } + // Control characters never reach the operator's terminal. + if reps[0].Detail != "bad[2Jstuff\nline two" || reps[0].Contact != "a@b c" { + t.Fatalf("report text not cleaned: %q %q", reps[0].Detail, reps[0].Contact) + } + if ok, _ := g.st.RemoveShare(bMade.ID, "takedown: test"); !ok { + t.Fatal("takedown removed nothing") + } + _, out = get(t, g.http.URL+"/v1/shares/"+bMade.ID) + if !strings.Contains(string(out), "taken_down") { + t.Fatalf("taken-down fetch: %s", out) + } +} + +func TestShareLimits(t *testing.T) { + w := newWorld(t) + g := w.global + g.srv.cfg.Share.Enabled = true + g.srv.cfg.Share.ViewerURL = "https://share.example" + a := w.newStem(g, "a@gmail.test") + a.register() + + g.srv.cfg.Share.QuotaBytes = 100 + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares", blob(60), true); code != 200 { + t.Fatalf("first upload answered %d", code) + } + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares", blob(60), true); code != 507 { + t.Fatalf("over quota answered %d", code) + } + g.srv.cfg.Share.QuotaBytes = 0 + g.srv.cfg.Share.PerAddressDay = 2 + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares", blob(60), true); code != 200 { + t.Fatalf("second upload answered %d", code) + } + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares", blob(60), true); code != 429 { + t.Fatalf("over the daily limit answered %d", code) + } + + g.srv.cfg.Share.PerAddressDay = 0 + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares?ttl=31", blob(60), true); code != 400 { + t.Fatalf("31-day share answered %d", code) + } + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares?ttl=30", blob(60), true); code != 200 { + t.Fatalf("30-day share answered %d", code) + } + + // At most max_live working links; revoking one frees a slot. + g.srv.cfg.Share.MaxLive = 3 // the three uploads above + if code, out := a.doRaw(g.http.URL, "POST", "/v1/shares?ttl=1", blob(60), true); code != 507 || !strings.Contains(string(out), "too_many_links") { + t.Fatalf("fourth live link answered %d %s", code, out) + } + _, out := a.do(g.http.URL, "GET", "/v1/shares", nil, true) + var list struct{ Shares []struct{ ID string } } + _ = json.Unmarshal(out, &list) + if len(list.Shares) != 3 { + t.Fatalf("list: %s", out) + } + a.do(g.http.URL, "DELETE", "/v1/shares/"+list.Shares[0].ID, nil, true) + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares?ttl=1", blob(60), true); code != 200 { + t.Fatalf("after a revoke answered %d", code) + } + + // A suspended address can't share. + _ = g.st.Suspend("a@gmail.test") + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares?ttl=7", blob(60), true); code != 403 { + t.Fatalf("suspended upload answered %d", code) + } +} + +func TestSharePreviewCard(t *testing.T) { + w := newWorld(t) + g := w.global + dir := t.TempDir() + page := "\n\nShared from Stem\n\n" + if err := os.WriteFile(filepath.Join(dir, "index.html"), []byte(page), 0o600); err != nil { + t.Fatal(err) + } + g.srv.cfg.Share.Enabled = true + g.srv.cfg.Share.ViewerURL = "http://" + strings.TrimPrefix(g.http.URL, "http://") + g.srv.cfg.Share.ViewerDir = dir + if err := os.MkdirAll(filepath.Join(dir, "assets"), 0o700); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(filepath.Join(dir, "assets", "index-abc.js"), []byte("console.log(1)"), 0o600) + a := w.newStem(g, "a@gmail.test") + a.register() + + card := base64.RawURLEncoding.EncodeToString([]byte(`{"t":"Terasa \"olej\"\u001b[2J","d":"Dve tenké\nvrstvy"}`)) + code, out := a.doRaw(g.http.URL, "POST", "/v1/shares?card="+card, blob(60), true) + if code != 200 { + t.Fatalf("upload: %d %s", code, out) + } + var made struct{ ID string } + _ = json.Unmarshal(out, &made) + + resp, err := http.Get(g.http.URL + "/s/" + made.ID) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + html := string(body) + if !strings.Contains(resp.Header.Get("X-Robots-Tag"), "noindex") { + t.Fatalf("no noindex header: %v", resp.Header) + } + for _, want := range []string{ + `Terasa <b>"olej"</b>[2J`, + `content="Dve tenké vrstvy"`, + `content="` + g.http.URL + `/s/` + made.ID + `"`, + `content="` + g.http.URL + `/og.png"`, + } { + if !strings.Contains(html, want) { + t.Fatalf("page lacks %s:\n%s", want, html) + } + } + if strings.Contains(html, "") || strings.Contains(html, "\x1b") { + t.Fatalf("card not escaped:\n%s", html) + } + + // No card: the page as built. Revoked: a removed card, the old text gone. + _, out = a.doRaw(g.http.URL, "POST", "/v1/shares", blob(60), true) + var plain struct{ ID string } + _ = json.Unmarshal(out, &plain) + if _, b := get(t, g.http.URL+"/s/"+plain.ID); string(b) != page { + t.Fatalf("cardless page changed:\n%s", b) + } + a.do(g.http.URL, "DELETE", "/v1/shares/"+made.ID, nil, true) + if _, b := get(t, g.http.URL+"/s/"+made.ID); strings.Contains(string(b), "Terasa") || !strings.Contains(string(b), "Link removed") { + t.Fatalf("revoked page:\n%s", b) + } + if code, _ := a.doRaw(g.http.URL, "POST", "/v1/shares?card=***", blob(60), true); code != 400 { + t.Fatalf("bad card answered %d", code) + } + + // Assets come from the viewer folder, with a long cache; nothing outside it. + resp, _ = http.Get(g.http.URL + "/assets/index-abc.js") + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || string(b) != "console.log(1)" || !strings.Contains(resp.Header.Get("Cache-Control"), "immutable") { + t.Fatalf("asset: %d %q %v", resp.StatusCode, b, resp.Header) + } + if code, _ := get(t, g.http.URL+"/assets/../index.html"); code == 200 { + t.Fatal("served outside assets") + } + + // Another host (the relay's own name) gets no viewer at all. + g.srv.cfg.Share.ViewerURL = "https://share.example" + if code, _ := get(t, g.http.URL+"/s/"+plain.ID); code != 404 { + t.Fatalf("viewer on the relay host answered %d", code) + } + if code, _ := get(t, g.http.URL+"/assets/index-abc.js"); code != 404 { + t.Fatalf("asset on the relay host answered %d", code) + } +} diff --git a/relay/internal/store/data.go b/relay/internal/store/data.go new file mode 100644 index 00000000..bae7c0c7 --- /dev/null +++ b/relay/internal/store/data.go @@ -0,0 +1,520 @@ +package store + +import ( + "database/sql" + "encoding/json" + "errors" + "strings" +) + +// ---- accounts and addresses ---- + +type Account struct { + Fp string + AuthKey string + KeyBlob string + Unlisted bool + Revoked bool +} + +func (s *Store) Account(fp string) (*Account, error) { + a := &Account{Fp: fp} + var u, r int + err := s.db.QueryRow(`SELECT auth_key, key_blob, unlisted, revoked FROM accounts WHERE fp = ?`, fp). + Scan(&a.AuthKey, &a.KeyBlob, &u, &r) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + a.Unlisted, a.Revoked = u == 1, r == 1 + return a, err +} + +// AddressAccount returns the account currently holding the address. +func (s *Store) AddressAccount(address string) (*Account, int64, error) { + var fp string + var verified int64 + err := s.db.QueryRow(`SELECT fp, verified FROM addresses WHERE address = ?`, address).Scan(&fp, &verified) + if errors.Is(err, sql.ErrNoRows) { + return nil, 0, ErrNotFound + } + if err != nil { + return nil, 0, err + } + a, err := s.Account(fp) + return a, verified, err +} + +func (s *Store) AccountAddresses(fp string) ([]string, error) { + rows, err := s.db.Query(`SELECT address FROM addresses WHERE fp = ? ORDER BY address`, fp) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var a string + if err := rows.Scan(&a); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +func (s *Store) SetUnlisted(fp string, v bool) error { + _, err := s.db.Exec(`UPDATE accounts SET unlisted = ? WHERE fp = ?`, b2i(v), fp) + return err +} + +// Revoke stops certifying the key and releases its addresses. +func (s *Store) Revoke(fp string) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + if _, err := tx.Exec(`UPDATE accounts SET revoked = 1 WHERE fp = ?`, fp); err != nil { + tx.Rollback() + return err + } + if _, err := tx.Exec(`DELETE FROM addresses WHERE fp = ?`, fp); err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +// ---- pending verifications ---- + +type Pending struct { + Address, Fp, AuthKey, KeyBlob, CodeHash, Token string + Created int64 + Attempts int +} + +func (s *Store) PutPending(p Pending) error { + _, err := s.db.Exec(`INSERT INTO pending (address, fp, auth_key, key_blob, code_hash, token, created, attempts) + VALUES (?, ?, ?, ?, ?, ?, ?, 0) + ON CONFLICT (address, fp) DO UPDATE SET auth_key = excluded.auth_key, key_blob = excluded.key_blob, + code_hash = excluded.code_hash, token = excluded.token, created = excluded.created, attempts = 0`, + p.Address, p.Fp, p.AuthKey, p.KeyBlob, p.CodeHash, p.Token, now()) + return err +} + +func (s *Store) scanPending(row *sql.Row) (*Pending, error) { + p := &Pending{} + err := row.Scan(&p.Address, &p.Fp, &p.AuthKey, &p.KeyBlob, &p.CodeHash, &p.Token, &p.Created, &p.Attempts) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + return p, err +} + +const pendingCols = `address, fp, auth_key, key_blob, code_hash, token, created, attempts` + +func (s *Store) GetPending(address, fp string) (*Pending, error) { + return s.scanPending(s.db.QueryRow(`SELECT `+pendingCols+` FROM pending WHERE address = ? AND fp = ?`, address, fp)) +} + +func (s *Store) GetPendingByToken(token string) (*Pending, error) { + return s.scanPending(s.db.QueryRow(`SELECT `+pendingCols+` FROM pending WHERE token = ?`, token)) +} + +// TakePendingAttempt spends one verification attempt; false when none are +// left. Atomic, so concurrent guesses can't overspend. +func (s *Store) TakePendingAttempt(address, fp string, max int) (bool, error) { + res, err := s.db.Exec(`UPDATE pending SET attempts = attempts + 1 WHERE address = ? AND fp = ? AND attempts < ?`, address, fp, max) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n == 1, err +} + +// CompleteVerification binds the address to the pending key, creating or +// refreshing the account, and drops every pending row for the address. +func (s *Store) CompleteVerification(p *Pending) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`INSERT INTO accounts (fp, auth_key, key_blob, created) VALUES (?, ?, ?, ?) + ON CONFLICT (fp) DO UPDATE SET key_blob = excluded.key_blob, revoked = 0`, p.Fp, p.AuthKey, p.KeyBlob, now()); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO addresses (address, fp, verified) VALUES (?, ?, ?) + ON CONFLICT (address) DO UPDATE SET fp = excluded.fp, verified = excluded.verified`, p.Address, p.Fp, now()); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM pending WHERE address = ?`, p.Address); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM suppressed WHERE address = ?`, p.Address); err != nil { + return err + } + return tx.Commit() +} + +// ---- suppression ---- + +func (s *Store) Suppressed(address string) (bool, error) { + var x int + err := s.db.QueryRow(`SELECT 1 FROM suppressed WHERE address = ?`, address).Scan(&x) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +func (s *Store) Suppress(address string) error { + _, err := s.db.Exec(`INSERT OR IGNORE INTO suppressed (address, at) VALUES (?, ?)`, address, now()) + return err +} + +func (s *Store) PutUnsubToken(token, address string) error { + _, err := s.db.Exec(`INSERT INTO unsub_tokens (token, address, at) VALUES (?, ?, ?)`, token, address, now()) + return err +} + +func (s *Store) UnsubToken(token string) (string, error) { + var a string + err := s.db.QueryRow(`SELECT address FROM unsub_tokens WHERE token = ?`, token).Scan(&a) + if errors.Is(err, sql.ErrNoRows) { + return "", ErrNotFound + } + return a, err +} + +// ---- allow / block / suspend ---- + +func (s *Store) Allow(fp, address string) error { + _, err := s.db.Exec(`INSERT OR IGNORE INTO allow (fp, address) VALUES (?, ?)`, fp, address) + return err +} + +func (s *Store) Unallow(fp, address string) error { + _, err := s.db.Exec(`DELETE FROM allow WHERE fp = ? AND address = ?`, fp, address) + return err +} + +func (s *Store) Allows(fp, address string) (bool, error) { + var x int + err := s.db.QueryRow(`SELECT 1 FROM allow WHERE fp = ? AND address = ?`, fp, address).Scan(&x) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +func (s *Store) Block(fp, address string) error { + _, err := s.db.Exec(`INSERT OR IGNORE INTO block (fp, address, at) VALUES (?, ?, ?)`, fp, address, now()) + return err +} + +func (s *Store) Unblock(fp, address string) error { + _, err := s.db.Exec(`DELETE FROM block WHERE fp = ? AND address = ?`, fp, address) + return err +} + +func (s *Store) Blocks(fp, address string) (bool, error) { + var x int + err := s.db.QueryRow(`SELECT 1 FROM block WHERE fp = ? AND address = ?`, fp, address).Scan(&x) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +// DistinctBlockers counts accounts that blocked address since `since`. +func (s *Store) DistinctBlockers(address string, since int64) (int, error) { + var n int + err := s.db.QueryRow(`SELECT COUNT(DISTINCT fp) FROM block WHERE address = ? AND at >= ?`, address, since).Scan(&n) + return n, err +} + +func (s *Store) Suspend(address string) error { + _, err := s.db.Exec(`INSERT OR IGNORE INTO suspended (address, at) VALUES (?, ?)`, address, now()) + return err +} + +func (s *Store) Unsuspend(address string) error { + _, err := s.db.Exec(`DELETE FROM suspended WHERE address = ?`, address) + return err +} + +func (s *Store) Suspended(address string) (bool, error) { + var x int + err := s.db.QueryRow(`SELECT 1 FROM suspended WHERE address = ?`, address).Scan(&x) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +// ---- held mail ---- + +type Held struct { + ID, To, From string + Size int64 + Envelope []byte +} + +func (s *Store) QueueUsage(to string) (bytes int64, mails int, err error) { + err = s.db.QueryRow(`SELECT COALESCE(SUM(size), 0), COUNT(*) FROM mail WHERE to_addr = ?`, to).Scan(&bytes, &mails) + return +} + +func (s *Store) QueueUsageFrom(to, from string) (bytes int64, mails int, err error) { + err = s.db.QueryRow(`SELECT COALESCE(SUM(size), 0), COUNT(*) FROM mail WHERE to_addr = ? AND from_addr = ?`, to, from).Scan(&bytes, &mails) + return +} + +// Seen reports whether an envelope id was already accepted (replay). +func (s *Store) Seen(id string) (bool, error) { + var x int + err := s.db.QueryRow(`SELECT 1 FROM seen WHERE id = ?`, id).Scan(&x) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +// MarkSeen records an envelope id. status is 'pending' for held mail and +// 'dropped' for mail the relay accepted but threw away. +func (s *Store) MarkSeen(id, signerFp, from, to, status string) error { + _, err := s.db.Exec(`INSERT INTO seen (id, signer_fp, from_addr, to_addr, at, status) VALUES (?, ?, ?, ?, ?, ?)`, + id, signerFp, from, to, now(), status) + return err +} + +func (s *Store) SeenRow(id string) (signerFp, from, to, status string, err error) { + err = s.db.QueryRow(`SELECT signer_fp, from_addr, to_addr, status FROM seen WHERE id = ?`, id).Scan(&signerFp, &from, &to, &status) + if errors.Is(err, sql.ErrNoRows) { + err = ErrNotFound + } + return +} + +func (s *Store) Hold(h Held, expires int64) error { + _, err := s.db.Exec(`INSERT INTO mail (id, to_addr, from_addr, size, envelope, received, expires) VALUES (?, ?, ?, ?, ?, ?, ?)`, + h.ID, h.To, h.From, h.Size, h.Envelope, now(), expires) + return err +} + +func (s *Store) HeldFor(addresses []string) ([]Held, error) { + if len(addresses) == 0 { + return nil, nil + } + q := `SELECT id, to_addr, from_addr, size, envelope FROM mail WHERE to_addr IN (?` + strings.Repeat(",?", len(addresses)-1) + `) ORDER BY received` + args := make([]any, len(addresses)) + for i, a := range addresses { + args[i] = a + } + rows, err := s.db.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Held + for rows.Next() { + var h Held + if err := rows.Scan(&h.ID, &h.To, &h.From, &h.Size, &h.Envelope); err != nil { + return nil, err + } + out = append(out, h) + } + return out, rows.Err() +} + +// Ack deletes held mail addressed to one of addresses and marks it +// delivered. Ids for other addresses are ignored. +func (s *Store) Ack(ids, addresses []string) (int, error) { + tx, err := s.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + owned := map[string]bool{} + for _, a := range addresses { + owned[a] = true + } + n := 0 + for _, id := range ids { + var to string + if err := tx.QueryRow(`SELECT to_addr FROM mail WHERE id = ?`, id).Scan(&to); err != nil { + continue + } + if !owned[to] { + continue + } + if _, err := tx.Exec(`DELETE FROM mail WHERE id = ?`, id); err != nil { + return 0, err + } + if _, err := tx.Exec(`UPDATE seen SET status = 'delivered' WHERE id = ?`, id); err != nil { + return 0, err + } + n++ + } + return n, tx.Commit() +} + +// ---- knocks ---- + +// Knock records an outstanding knock from → to. It returns false when one +// is already outstanding (newer than since). +func (s *Store) Knock(from, to string, since int64) (bool, error) { + var at int64 + err := s.db.QueryRow(`SELECT at FROM knocks WHERE from_addr = ? AND to_addr = ?`, from, to).Scan(&at) + if err == nil && at > since { + return false, nil + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return false, err + } + _, err = s.db.Exec(`INSERT INTO knocks (from_addr, to_addr, at) VALUES (?, ?, ?) + ON CONFLICT (from_addr, to_addr) DO UPDATE SET at = excluded.at`, from, to, now()) + return err == nil, err +} + +// ClearKnock forgets an outstanding knock (the recipient accepted). +func (s *Store) ClearKnock(from, to string) error { + _, err := s.db.Exec(`DELETE FROM knocks WHERE from_addr = ? AND to_addr = ?`, from, to) + return err +} + +// ---- invites ---- + +func (s *Store) InvitesTo(to string, since int64) (count int, last int64, err error) { + err = s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(at), 0) FROM invite_log WHERE to_addr = ?`, to).Scan(&count, &last) + _ = since + return +} + +func (s *Store) QueueInvite(to, from string) error { + var raw string + err := s.db.QueryRow(`SELECT senders FROM invite_queue WHERE to_addr = ?`, to).Scan(&raw) + var senders []string + if err == nil { + _ = json.Unmarshal([]byte(raw), &senders) + } else if !errors.Is(err, sql.ErrNoRows) { + return err + } + for _, x := range senders { + if x == from { + return nil + } + } + senders = append(senders, from) + b, _ := json.Marshal(senders) + _, err = s.db.Exec(`INSERT INTO invite_queue (to_addr, senders, queued) VALUES (?, ?, ?) + ON CONFLICT (to_addr) DO UPDATE SET senders = excluded.senders`, to, string(b), now()) + return err +} + +type QueuedInvite struct { + To string + Senders []string + Queued int64 +} + +func (s *Store) InviteQueue(limit int) ([]QueuedInvite, error) { + rows, err := s.db.Query(`SELECT to_addr, senders, queued FROM invite_queue ORDER BY queued LIMIT ?`, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []QueuedInvite + for rows.Next() { + var q QueuedInvite + var raw string + if err := rows.Scan(&q.To, &raw, &q.Queued); err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(raw), &q.Senders) + out = append(out, q) + } + return out, rows.Err() +} + +func (s *Store) DropQueuedInvite(to string) error { + _, err := s.db.Exec(`DELETE FROM invite_queue WHERE to_addr = ?`, to) + return err +} + +func (s *Store) DropQueuedInvitesBefore(t int64) error { + _, err := s.db.Exec(`DELETE FROM invite_queue WHERE queued <= ?`, t) + return err +} + +func (s *Store) LogInvite(to, tag string, senders []string) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`INSERT INTO invite_log (to_addr, at, tag) VALUES (?, ?, ?)`, to, now(), tag); err != nil { + return err + } + for _, f := range senders { + if _, err := tx.Exec(`INSERT INTO invite_senders (tag, from_addr) VALUES (?, ?)`, tag, f); err != nil { + return err + } + } + if _, err := tx.Exec(`DELETE FROM invite_queue WHERE to_addr = ?`, to); err != nil { + return err + } + return tx.Commit() +} + +func (s *Store) InviteTagAddress(tag string) (to string, senders []string, err error) { + if err = s.db.QueryRow(`SELECT to_addr FROM invite_log WHERE tag = ?`, tag).Scan(&to); err != nil { + if errors.Is(err, sql.ErrNoRows) { + err = ErrNotFound + } + return + } + rows, err := s.db.Query(`SELECT from_addr FROM invite_senders WHERE tag = ?`, tag) + if err != nil { + return + } + defer rows.Close() + for rows.Next() { + var f string + if err = rows.Scan(&f); err != nil { + return + } + senders = append(senders, f) + } + err = rows.Err() + return +} + +func (s *Store) BanInviter(from string) error { + _, err := s.db.Exec(`INSERT OR IGNORE INTO invite_banned (from_addr, at) VALUES (?, ?)`, from, now()) + return err +} + +func (s *Store) InviterBanned(from string) (bool, error) { + var x int + err := s.db.QueryRow(`SELECT 1 FROM invite_banned WHERE from_addr = ?`, from).Scan(&x) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +func (s *Store) LogMailEvent(tag, kind string) error { + _, err := s.db.Exec(`INSERT INTO mail_events (tag, kind, at) VALUES (?, ?, ?)`, tag, kind, now()) + return err +} + +// InviteRates returns invites sent and bounce/complaint counts since t. +func (s *Store) InviteRates(since int64) (sent, bounces, complaints int, err error) { + if err = s.db.QueryRow(`SELECT COUNT(*) FROM invite_log WHERE at >= ?`, since).Scan(&sent); err != nil { + return + } + err = s.db.QueryRow(`SELECT + COALESCE(SUM(kind = 'bounce'), 0), COALESCE(SUM(kind = 'complaint'), 0) + FROM mail_events e WHERE at >= ? AND EXISTS (SELECT 1 FROM invite_log l WHERE l.tag = e.tag)`, since).Scan(&bounces, &complaints) + return +} diff --git a/relay/internal/store/shares.go b/relay/internal/store/shares.go new file mode 100644 index 00000000..f382dc3d --- /dev/null +++ b/relay/internal/store/shares.go @@ -0,0 +1,125 @@ +package store + +import ( + "database/sql" + "errors" +) + +// Share is one public share link's row. Blob is nil once removed. +type Share struct { + ID string + Owner string + Fp string + Size int64 + Blob []byte + Created int64 + Expires int64 // 0 = never + Removed int64 // 0 = live + RemovedWhy string + // The link preview the sharer chose to publish in the clear (may be empty). + CardTitle string + CardDesc string +} + +func (s *Store) PutShare(sh *Share) error { + _, err := s.db.Exec(`INSERT INTO shares (id, owner, fp, size, blob, created, expires, card_title, card_desc) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + sh.ID, sh.Owner, sh.Fp, sh.Size, sh.Blob, sh.Created, sh.Expires, sh.CardTitle, sh.CardDesc) + return err +} + +// GetShare returns the row with its blob. An expired share not yet swept +// reads as not found. +func (s *Store) GetShare(id string) (*Share, error) { + var sh Share + err := s.db.QueryRow(`SELECT id, owner, fp, size, blob, created, expires, removed, removed_why, card_title, card_desc + FROM shares WHERE id = ?`, id). + Scan(&sh.ID, &sh.Owner, &sh.Fp, &sh.Size, &sh.Blob, &sh.Created, &sh.Expires, &sh.Removed, &sh.RemovedWhy, + &sh.CardTitle, &sh.CardDesc) + if errors.Is(err, sql.ErrNoRows) || err == nil && sh.Expires > 0 && sh.Expires <= now() { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + return &sh, nil +} + +// RemoveShare drops the blob and keeps a tombstone. It reports whether a +// live share was removed. +func (s *Store) RemoveShare(id, why string) (bool, error) { + res, err := s.db.Exec(`UPDATE shares SET blob = NULL, card_title = '', card_desc = '', removed = ?, removed_why = ? + WHERE id = ? AND removed = 0`, now(), why, id) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n == 1, err +} + +// OwnerShares lists an address's live shares, newest first, without blobs. +func (s *Store) OwnerShares(owner string) ([]Share, error) { + rows, err := s.db.Query(`SELECT id, size, created, expires FROM shares + WHERE owner = ? AND removed = 0 AND (expires = 0 OR expires > ?) ORDER BY created DESC`, owner, now()) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Share + for rows.Next() { + sh := Share{Owner: owner} + if err := rows.Scan(&sh.ID, &sh.Size, &sh.Created, &sh.Expires); err != nil { + return nil, err + } + out = append(out, sh) + } + return out, rows.Err() +} + +// OwnerShareUsage is how many live shares an address holds and their bytes. +func (s *Store) OwnerShareUsage(owner string) (count int, bytes int64, err error) { + err = s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size), 0) FROM shares WHERE owner = ? AND removed = 0 AND (expires = 0 OR expires > ?)`, + owner, now()).Scan(&count, &bytes) + return count, bytes, err +} + +type ShareReport struct { + ID int64 + Share string + Reason string + Detail string + Contact string + Key string + IP string + At int64 +} + +func (s *Store) AddShareReport(r *ShareReport) error { + _, err := s.db.Exec(`INSERT INTO share_reports (share, reason, detail, contact, key, ip, at) VALUES (?, ?, ?, ?, ?, ?, ?)`, + r.Share, r.Reason, r.Detail, r.Contact, r.Key, r.IP, now()) + return err +} + +// OpenShareReports lists reports not yet marked handled, oldest first. +func (s *Store) OpenShareReports() ([]ShareReport, error) { + rows, err := s.db.Query(`SELECT id, share, reason, detail, contact, key, ip, at FROM share_reports WHERE handled = 0 ORDER BY at`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []ShareReport + for rows.Next() { + var r ShareReport + if err := rows.Scan(&r.ID, &r.Share, &r.Reason, &r.Detail, &r.Contact, &r.Key, &r.IP, &r.At); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// HandleShareReports marks every report on a share handled. +func (s *Store) HandleShareReports(share string) error { + _, err := s.db.Exec(`UPDATE share_reports SET handled = 1 WHERE share = ?`, share) + return err +} diff --git a/relay/internal/store/store.go b/relay/internal/store/store.go new file mode 100644 index 00000000..3261c03f --- /dev/null +++ b/relay/internal/store/store.go @@ -0,0 +1,255 @@ +// Package store is the relay's SQLite file. Migrations only go forward. +package store + +import ( + "database/sql" + "errors" + "fmt" + "time" + + _ "modernc.org/sqlite" +) + +type Store struct{ db *sql.DB } + +var migrations = []string{ + `CREATE TABLE relay_keys (kid TEXT PRIMARY KEY, priv BLOB NOT NULL, created INTEGER NOT NULL, active INTEGER NOT NULL); + CREATE TABLE accounts (fp TEXT PRIMARY KEY, auth_key TEXT NOT NULL, key_blob TEXT NOT NULL, + unlisted INTEGER NOT NULL DEFAULT 0, revoked INTEGER NOT NULL DEFAULT 0, created INTEGER NOT NULL); + CREATE TABLE addresses (address TEXT PRIMARY KEY, fp TEXT NOT NULL, verified INTEGER NOT NULL); + CREATE INDEX addresses_fp ON addresses(fp); + CREATE TABLE pending (address TEXT NOT NULL, fp TEXT NOT NULL, auth_key TEXT NOT NULL, key_blob TEXT NOT NULL, + code_hash TEXT NOT NULL, token TEXT NOT NULL UNIQUE, created INTEGER NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (address, fp)); + CREATE TABLE suppressed (address TEXT PRIMARY KEY, at INTEGER NOT NULL); + CREATE TABLE unsub_tokens (token TEXT PRIMARY KEY, address TEXT NOT NULL, at INTEGER NOT NULL); + CREATE TABLE allow (fp TEXT NOT NULL, address TEXT NOT NULL, PRIMARY KEY (fp, address)); + CREATE TABLE block (fp TEXT NOT NULL, address TEXT NOT NULL, at INTEGER NOT NULL, PRIMARY KEY (fp, address)); + CREATE INDEX block_address ON block(address, at); + CREATE TABLE suspended (address TEXT PRIMARY KEY, at INTEGER NOT NULL); + CREATE TABLE mail (id TEXT PRIMARY KEY, to_addr TEXT NOT NULL, from_addr TEXT NOT NULL, size INTEGER NOT NULL, + envelope BLOB NOT NULL, received INTEGER NOT NULL, expires INTEGER NOT NULL); + CREATE INDEX mail_to ON mail(to_addr, received); + CREATE TABLE seen (id TEXT PRIMARY KEY, signer_fp TEXT NOT NULL, from_addr TEXT NOT NULL, to_addr TEXT NOT NULL, + at INTEGER NOT NULL, status TEXT NOT NULL); + CREATE TABLE knocks (from_addr TEXT NOT NULL, to_addr TEXT NOT NULL, at INTEGER NOT NULL, PRIMARY KEY (from_addr, to_addr)); + CREATE TABLE counters (k TEXT NOT NULL, bucket INTEGER NOT NULL, n INTEGER NOT NULL, expires INTEGER NOT NULL, PRIMARY KEY (k, bucket)); + CREATE TABLE invite_queue (to_addr TEXT PRIMARY KEY, senders TEXT NOT NULL, queued INTEGER NOT NULL); + CREATE TABLE invite_log (to_addr TEXT NOT NULL, at INTEGER NOT NULL, tag TEXT NOT NULL); + CREATE INDEX invite_log_to ON invite_log(to_addr, at); + CREATE TABLE invite_senders (tag TEXT NOT NULL, from_addr TEXT NOT NULL); + CREATE TABLE invite_banned (from_addr TEXT PRIMARY KEY, at INTEGER NOT NULL); + CREATE TABLE mail_events (tag TEXT NOT NULL, kind TEXT NOT NULL, at INTEGER NOT NULL); + CREATE TABLE meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);`, + `CREATE TABLE push_tokens (token TEXT PRIMARY KEY, fp TEXT NOT NULL, used INTEGER NOT NULL);`, + `CREATE TABLE shares (id TEXT PRIMARY KEY, owner TEXT NOT NULL, fp TEXT NOT NULL, size INTEGER NOT NULL, blob BLOB, + created INTEGER NOT NULL, expires INTEGER NOT NULL, removed INTEGER NOT NULL DEFAULT 0, removed_why TEXT NOT NULL DEFAULT ''); + CREATE INDEX shares_owner ON shares(owner, created); + CREATE TABLE share_reports (id INTEGER PRIMARY KEY AUTOINCREMENT, share TEXT NOT NULL, reason TEXT NOT NULL, + detail TEXT NOT NULL, contact TEXT NOT NULL, key TEXT NOT NULL, ip TEXT NOT NULL, at INTEGER NOT NULL, + handled INTEGER NOT NULL DEFAULT 0);`, + `ALTER TABLE shares ADD COLUMN card_title TEXT NOT NULL DEFAULT ''; + ALTER TABLE shares ADD COLUMN card_desc TEXT NOT NULL DEFAULT '';`, +} + +func Open(path string) (*Store, error) { + db, err := sql.Open("sqlite", "file:"+path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)") + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + s := &Store{db: db} + return s, s.migrate() +} + +func (s *Store) Close() error { return s.db.Close() } + +func (s *Store) DB() *sql.DB { return s.db } + +func (s *Store) migrate() error { + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS schema (version INTEGER NOT NULL)`); err != nil { + return err + } + var v int + if err := s.db.QueryRow(`SELECT COALESCE(MAX(version), 0) FROM schema`).Scan(&v); err != nil { + return err + } + if v > len(migrations) { + return fmt.Errorf("database schema %d is newer than this relay (%d); upgrade the relay", v, len(migrations)) + } + for i := v; i < len(migrations); i++ { + tx, err := s.db.Begin() + if err != nil { + return err + } + if _, err := tx.Exec(migrations[i]); err != nil { + tx.Rollback() + return fmt.Errorf("migration %d: %w", i+1, err) + } + if _, err := tx.Exec(`INSERT INTO schema (version) VALUES (?)`, i+1); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +var ErrNotFound = errors.New("not found") + +func now() int64 { return time.Now().UnixMilli() } + +// ---- relay keys ---- + +type RelayKey struct { + Kid string + Priv []byte + Active bool +} + +func (s *Store) RelayKeys() ([]RelayKey, error) { + rows, err := s.db.Query(`SELECT kid, priv, active FROM relay_keys ORDER BY created`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []RelayKey + for rows.Next() { + var k RelayKey + var a int + if err := rows.Scan(&k.Kid, &k.Priv, &a); err != nil { + return nil, err + } + k.Active = a == 1 + out = append(out, k) + } + return out, rows.Err() +} + +func (s *Store) AddRelayKey(kid string, priv []byte, active bool) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + if active { + if _, err := tx.Exec(`UPDATE relay_keys SET active = 0`); err != nil { + tx.Rollback() + return err + } + } + if _, err := tx.Exec(`INSERT INTO relay_keys (kid, priv, created, active) VALUES (?, ?, ?, ?)`, kid, priv, now(), b2i(active)); err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (s *Store) ActivateRelayKey(kid string) error { + _, err := s.db.Exec(`UPDATE relay_keys SET active = (kid = ?)`, kid) + return err +} + +func (s *Store) RemoveRelayKey(kid string) error { + _, err := s.db.Exec(`DELETE FROM relay_keys WHERE kid = ? AND active = 0`, kid) + return err +} + +func b2i(b bool) int { + if b { + return 1 + } + return 0 +} + +// ---- counters (fixed windows) ---- + +// Hit increments k in the current window and reports whether the count is +// still within limit. limit <= 0 means unlimited (but still counted). +func (s *Store) Hit(k string, window time.Duration, limit int) (bool, error) { + w := window.Milliseconds() + bucket := time.Now().UnixMilli() / w + var n int + err := s.db.QueryRow(`INSERT INTO counters (k, bucket, n, expires) VALUES (?, ?, 1, ?) + ON CONFLICT (k, bucket) DO UPDATE SET n = n + 1 RETURNING n`, k, bucket, (bucket+1)*w).Scan(&n) + if err != nil { + return false, err + } + return limit <= 0 || n <= limit, nil +} + +// Count reads k in the current window without incrementing. +func (s *Store) Count(k string, window time.Duration) (int, error) { + bucket := time.Now().UnixMilli() / window.Milliseconds() + var n int + err := s.db.QueryRow(`SELECT n FROM counters WHERE k = ? AND bucket = ?`, k, bucket).Scan(&n) + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + return n, err +} + +func (s *Store) Meta(k string) (string, error) { + var v string + err := s.db.QueryRow(`SELECT v FROM meta WHERE k = ?`, k).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return v, err +} + +func (s *Store) SetMeta(k, v string) error { + _, err := s.db.Exec(`INSERT INTO meta (k, v) VALUES (?, ?) ON CONFLICT (k) DO UPDATE SET v = excluded.v`, k, v) + return err +} + +// Sweep deletes expired rows and marks expired held mail in seen. +func (s *Store) Sweep(holdDays int) error { + t := now() + if _, err := s.db.Exec(`UPDATE seen SET status = 'expired' WHERE status = 'pending' AND id IN (SELECT id FROM mail WHERE expires <= ?)`, t); err != nil { + return err + } + stmts := []struct { + q string + args []any + }{ + {`DELETE FROM mail WHERE expires <= ?`, []any{t}}, + {`DELETE FROM seen WHERE at <= ?`, []any{t - int64(holdDays+1)*86400000}}, + {`DELETE FROM pending WHERE created <= ?`, []any{t - 86400000}}, + {`DELETE FROM counters WHERE expires <= ?`, []any{t}}, + {`DELETE FROM unsub_tokens WHERE at <= ?`, []any{t - 400*86400000}}, + {`DELETE FROM push_tokens WHERE used <= ?`, []any{t - 90*86400000}}, + // An expired share is gone for good; a revoked one keeps its + // tombstone (so the link says "removed") for a year. + {`DELETE FROM shares WHERE expires > 0 AND expires <= ?`, []any{t}}, + {`DELETE FROM shares WHERE removed > 0 AND removed <= ?`, []any{t - 365*86400000}}, + } + for _, st := range stmts { + if _, err := s.db.Exec(st.q, st.args...); err != nil { + return err + } + } + return nil +} + +// ---- push tokens ---- + +// ClaimPushToken binds token to fp on first use and reports whether fp owns +// it. Another fp takes it over only once the owner has been idle for rebind. +func (s *Store) ClaimPushToken(token, fp string, rebind time.Duration) (bool, error) { + t := now() + res, err := s.db.Exec(`INSERT INTO push_tokens (token, fp, used) VALUES (?, ?, ?) + ON CONFLICT (token) DO UPDATE SET fp = excluded.fp, used = excluded.used + WHERE push_tokens.fp = excluded.fp OR push_tokens.used <= ?`, token, fp, t, t-rebind.Milliseconds()) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n == 1, err +} + +func (s *Store) DropPushToken(token string) error { + _, err := s.db.Exec(`DELETE FROM push_tokens WHERE token = ?`, token) + return err +} diff --git a/relay/testdata/vectors.json b/relay/testdata/vectors.json new file mode 100644 index 00000000..9a042740 --- /dev/null +++ b/relay/testdata/vectors.json @@ -0,0 +1,68 @@ +{ + "cert": { + "cert": { + "v": 1, + "address": "vlado@gmail.test", + "keyFp": "6a3803d5f059902a1c6dafbc9ba4729212f7caac08634cc3ae76b27529f03827", + "issuer": "https://relay.test", + "kid": "k1", + "iat": 1760000000000, + "exp": 1760086400000, + "sig": "aQ5YhvjWZWkntsA6jdrb67CeyDbYy6ZVtEVUi5HUhfNa37fgNyAoMM8mGUNgd91KAWqDt1J2MnqtOHJEZ2vSBg==" + }, + "signingString": "stem-cert-v1\nvlado@gmail.test\n6a3803d5f059902a1c6dafbc9ba4729212f7caac08634cc3ae76b27529f03827\nhttps://relay.test\n1760000000000\n1760086400000\nk1" + }, + "envelope": { + "envelope": { + "v": 1, + "id": "0123456789abcdef0123456789abcdef", + "ts": 1760000000000, + "from": "vlado@gmail.test", + "to": "martin@acme.test", + "cert": { + "v": 1, + "address": "vlado@gmail.test", + "keyFp": "6a3803d5f059902a1c6dafbc9ba4729212f7caac08634cc3ae76b27529f03827", + "issuer": "https://relay.test", + "kid": "k1", + "iat": 1760000000000, + "exp": 1760086400000, + "sig": "aQ5YhvjWZWkntsA6jdrb67CeyDbYy6ZVtEVUi5HUhfNa37fgNyAoMM8mGUNgd91KAWqDt1J2MnqtOHJEZ2vSBg==" + }, + "signer": "gTl3Dqh9F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5Q=", + "sealed": "rAGyIJ6GNU+4UyN7XeD0+rE8f8v0M6YcAZNpYX/s8Qua8uKYt5Y/9dv9P9yjny1jG/dy0C1tmQ2HQRNesxENNGz89e60I1KCzyTume52Wm7aoRXeESe2I2hDdmE=", + "sig": "qArl/QwEYaruTlZviGBxCR5A+nZ3uZPYvKxrKl0g29c1OjUHjQL56JbZM5/fHd8HmRYP17VgLqxP+f04jxLLBg==" + }, + "plaintext": "{\"v\":1,\"subject\":\"Hi\",\"body\":\"Hello Martin\"}", + "signingString": "stem-env-v1\n0123456789abcdef0123456789abcdef\n1760000000000\nvlado@gmail.test\nmartin@acme.test\n77dbd136fea07844749fbec4981164f2236f368eaec26deb407cf67a1cf10368\n95941c60650ff18bc4bd6dae5696f4864200442ce56ffa32e260b50de575c906" + }, + "keyBlob": { + "enc": "Xf7dO2vUf2+ijuFdlp1bsOpTd01Ii9r53xxuASSz7yI=", + "encSig": "RNV2v8xF56jjjHKLXzPrKA6kT6KAQq8UVIimCSHLEO9AOOqNgCP7McL0esJEVfgf6ZC2omk9tJcc+Jk6pNsZDw==" + }, + "keys": { + "ephemeralPriv": "0404040404040404040404040404040404040404040404040404040404040404", + "ephemeralPub": "rAGyIJ6GNU+4UyN7XeD0+rE8f8v0M6YcAZNpYX/s8Qs=", + "recipientEncPriv": "0303030303030303030303030303030303030303030303030303030303030303", + "recipientEncPub": "Xf7dO2vUf2+ijuFdlp1bsOpTd01Ii9r53xxuASSz7yI=", + "relayPub": "iojj3XQJ8ZX9UtstPLpdcspnCb8dlBIb83SIAbQPb1w=", + "relaySeed": "0101010101010101010101010101010101010101010101010101010101010101", + "stemFp": "6a3803d5f059902a1c6dafbc9ba4729212f7caac08634cc3ae76b27529f03827", + "stemPub": "gTl3Dqh9F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5Q=", + "stemSeed": "0202020202020202020202020202020202020202020202020202020202020202" + }, + "pow": { + "address": "vlado@gmail.test", + "bits": 8, + "key": "gTl3Dqh9F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5Q=", + "nonce": "247" + }, + "request": { + "body": "{\"address\":\"martin@acme.test\"}", + "method": "POST", + "path": "/v1/cert", + "sig": "vfdCX1bgSPItDSuDyqeVsAhrsd25iP15ViCXfy9cbEZBO9b5wlCAgCXaFc3WTXFdIAZ0MZ1pva48k7VR7mAdAQ==", + "signingString": "stem-req-v1\nPOST\n/v1/cert\n1760000000000\na7a7aca75482e2b8587b3a48840c894fbfd62b0a762f2646582f274a0a57ce13", + "time": 1760000000000 + } +} diff --git a/scripts/browser-extension-live.mjs b/scripts/browser-extension-live.mjs index 110f3676..9c0e1164 100644 --- a/scripts/browser-extension-live.mjs +++ b/scripts/browser-extension-live.mjs @@ -249,6 +249,8 @@ function fixture(req, res, state) { return send(204, 'text/plain', ''); case '/user': return html('The user’s own page', '

The user is reading this

Stem must not switch away from it.

'); + case '/extframe': + return html('Page with a fill menu', '

Billing address

'); case '/page2': return html('Page two', '

Second page

Nothing else here.

'); case '/help': @@ -390,6 +392,7 @@ async function writeHelperExtension(dir, origin, recordFixture) { permissions: ['debugger', 'tabs', 'downloads'], host_permissions: ['http://127.0.0.1/*'], background: { service_worker: 'sw.js' }, + web_accessible_resources: [{ resources: ['frame.html'], matches: ['http://127.0.0.1/*'] }], content_scripts: [{ matches: ['http://127.0.0.1/*'], js: ['cs.js'], run_at: 'document_start' }] }, null, @@ -413,7 +416,20 @@ async function writeHelperExtension(dir, origin, recordFixture) { report(); }; if (document.documentElement) start(); else document.addEventListener('readystatechange', start, { once: true }); + // A password manager's fill menu, as Bitwarden draws it: an element with a + // closed shadow root holding a frame of this extension's own page. A page + // with one refuses every other extension's debugger. + const foreignFrame = () => { + const host = document.createElement('test-fill-menu'); + host.attachShadow({ mode: 'closed' }).innerHTML = ''; + document.documentElement.append(host); + }; + if (location.pathname === '/extframe') addEventListener('DOMContentLoaded', foreignFrame); addEventListener('hashchange', () => { + if (location.hash === '#stem-test-foreign-frame') { + foreignFrame(); + return; + } if (['#stem-test-minimize', '#stem-test-restore', '#stem-test-user-download'].includes(location.hash)) { chrome.runtime.sendMessage({ op: location.hash.slice(11) }); return; @@ -427,6 +443,7 @@ async function writeHelperExtension(dir, origin, recordFixture) { }); })();\n` ); + await writeFile(join(dir, 'frame.html'), '

Fill menu

\n'); // Service worker: optionally captures a raw AX tree of the fixture form in a // tab of its own (never one the scenario uses) for tests/fixtures. await writeFile( @@ -587,7 +604,8 @@ async function nativeHost(configPath) { assert(hello.extensionId === config.extensionId, `id ${hello.extensionId} != ${config.extensionId}`); assert(hello.extensionVersion === config.expectedVersion, `version ${hello.extensionVersion}`); assert(/Chrome\//.test(hello.userAgent), 'userAgent'); - return `${hello.extensionId} v${hello.extensionVersion}`; + assert(/^[0-9a-f-]{36}$/.test(hello.profileId || ''), `profileId ${hello.profileId}`); + return `${hello.extensionId} v${hello.extensionVersion}, profile ${hello.profileId.slice(0, 8)}…`; }); sendFrame({ type: 'host', connected: true }); @@ -940,6 +958,23 @@ async function nativeHost(configPath) { return c.text; }); + await check('another extension’s frame in the page is taken out so Stem can attach; one added later too', async () => { + const D = 'thread-d'; + const opened = ok(await req(D, { kind: 'open', url: `${config.origin}/extframe` }), /Page with a fill menu/); + const s1 = ok(await req(D, { kind: 'snapshot' }), /Billing address/); + const menus = await evalIn(D, opened.tab, 'document.querySelectorAll("test-fill-menu").length'); + assert(menus === 1, `fill menu host count ${menus}`); + // The menu coming back while Stem is attached (the user focused a field). + await req(D, { kind: 'evaluate', script: 'location.hash = "stem-test-foreign-frame", 1' }); + await sleep(800); + // Chrome drops the debugger then; the next action attaches again, and the refs still work. + const s2 = ok(await req(D, { kind: 'fill', ref: refOf(s1.text, 'textbox', 'City'), value: 'Most' }), /Set textbox "City"/, 'fill after a later frame'); + const city = await evalIn(D, opened.tab, 'document.querySelector("#city").value'); + sendFrame({ type: 'end', threadId: D }); + assert(city === 'Most', `city ${city}`); + return s2.text.split('\n').at(-1); + }); + await check('stale and unknown refs say so', async () => { const r = await req(C, { kind: 'click', tab: formTab, ref: 'e3' }); assert(!r.ok && r.error === 'Ref e3 is stale: take a new snapshot.', JSON.stringify(r)); diff --git a/scripts/build-native-helpers.mjs b/scripts/build-native-helpers.mjs new file mode 100644 index 00000000..f7caaefd --- /dev/null +++ b/scripts/build-native-helpers.mjs @@ -0,0 +1,11 @@ +// electron-builder's beforePack: the computer-control helper for the platform +// being packed — the Swift one for a Mac (build-mac-helper.mjs), the C# one for +// Windows (build-windows-helper.mjs). Linux needs no build: its helper is a gjs +// script shipped as is (native/linux, see electron-builder.yml). +import { buildHelper as buildMac } from './build-mac-helper.mjs'; +import { buildHelper as buildWindows } from './build-windows-helper.mjs'; + +export default async function beforePack(context) { + if (context.electronPlatformName === 'darwin') buildMac(); + if (context.electronPlatformName === 'win32') buildWindows(); +} diff --git a/scripts/build-windows-helper.mjs b/scripts/build-windows-helper.mjs new file mode 100644 index 00000000..4bc82846 --- /dev/null +++ b/scripts/build-windows-helper.mjs @@ -0,0 +1,67 @@ +// Builds native/windows/stem-computer (the C# helper behind computer control +// and "Ask about this window" on Windows) into build/native/stem-computer.exe. +// +// The compiler is the one every Windows ships with .NET Framework 4.8 +// (Framework64\v4.0.30319\csc.exe, C# 5), so neither a dev machine nor CI needs +// an SDK. Two entry points like build-mac-helper.mjs: electron-builder's +// beforePack (scripts/build-native-helpers.mjs, Windows targets only) and a +// plain `node scripts/build-windows-helper.mjs [--output ]`, which the +// desktop app runs on demand in development (src/desktop/computer-host/helper.ts). +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +export const SOURCE_DIR = resolve(here, '..', 'native', 'windows', 'stem-computer'); +export const OUTPUT = resolve(here, '..', 'build', 'native', 'stem-computer.exe'); + +export function sourceFiles() { + return readdirSync(SOURCE_DIR) + .filter((f) => f.endsWith('.cs')) + .sort() + .map((f) => join(SOURCE_DIR, f)); +} + +/** A hash of the sources, so a dev build can tell whether its binary is current. */ +export function sourceHash() { + const h = createHash('sha256'); + for (const file of sourceFiles()) h.update(readFileSync(file)); + return h.digest('hex').slice(0, 16); +} + +function framework() { + const root = join(process.env.WINDIR ?? 'C:\\Windows', 'Microsoft.NET', 'Framework64', 'v4.0.30319'); + if (!existsSync(join(root, 'csc.exe'))) throw new Error(`No C# compiler at ${root} (.NET Framework 4.8 is part of Windows 10 and 11).`); + return root; +} + +export function buildHelper({ output = OUTPUT, log = console.log } = {}) { + const fw = framework(); + const refs = [ + 'System.dll', + 'System.Core.dll', + 'System.Drawing.dll', + 'System.Web.Extensions.dll', + join(fw, 'WPF', 'UIAutomationClient.dll'), + join(fw, 'WPF', 'UIAutomationTypes.dll'), + join(fw, 'WPF', 'WindowsBase.dll') + ]; + mkdirSync(dirname(output), { recursive: true }); + rmSync(output, { force: true }); + log(`[stem-computer] csc → ${output}`); + execFileSync( + join(fw, 'csc.exe'), + ['/nologo', '/optimize+', '/target:exe', '/platform:anycpu', `/out:${output}`, ...refs.map((r) => `/reference:${r}`), ...sourceFiles()], + { stdio: 'inherit', windowsHide: true } + ); + log(`[stem-computer] built ${output} (${statSync(output).size} bytes)`); + return output; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const args = process.argv.slice(2); + const at = args.indexOf('--output'); + buildHelper(at >= 0 && args[at + 1] ? { output: resolve(args[at + 1]) } : {}); +} diff --git a/scripts/drive-helper.mjs b/scripts/drive-helper.mjs new file mode 100644 index 00000000..3c30a7fa --- /dev/null +++ b/scripts/drive-helper.mjs @@ -0,0 +1,78 @@ +// Drives a computer-control helper (any platform's) through a list of commands +// from a JSON file and records what it answered: one line per reply in +// /log.txt, each picture as /-.jpg. For checking a helper +// by hand on a real desktop, e.g. the Windows VM's (scripts/windows-vm.mjs desktop). +// +// node scripts/drive-helper.mjs +// +// commands.json: [{"cmd": "list-windows"}, {"sleep": 500}, {"cmd": "click", "x": 10, "y": 20}, …] +// A string in place of a command runs that program first (to open a window to work on). +import { spawn } from 'node:child_process'; +import { mkdirSync, readFileSync, writeFileSync, appendFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { createInterface } from 'node:readline'; + +const [helperPath, commandsPath, out] = process.argv.slice(2); +if (!helperPath || !commandsPath || !out) { + console.error('usage: node scripts/drive-helper.mjs '); + process.exit(2); +} +mkdirSync(out, { recursive: true }); +const logFile = join(out, 'log.txt'); +writeFileSync(logFile, ''); +const log = (line) => { + appendFileSync(logFile, `${line}\n`); + console.log(line); +}; + +const helper = spawn(helperPath, [], { + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, STEM_COMPUTER_TRACE: '1', STEM_PARENT_PID: String(process.pid) } +}); +helper.stderr.on('data', (d) => log(`stderr ${d.toString().trim()}`)); +const lines = createInterface({ input: helper.stdout }); +const waiting = new Map(); +let nextId = 1; +lines.on('line', (line) => { + const reply = JSON.parse(line); + if (reply.event) return log(`event ${JSON.stringify(reply)}`); + waiting.get(reply.id)?.(reply); + waiting.delete(reply.id); +}); + +const call = (cmd, fields) => + new Promise((resolve) => { + const id = nextId++; + const timer = setTimeout(() => resolve({ ok: false, error: 'no answer in 60s' }), 60_000); + waiting.set(id, (r) => { + clearTimeout(timer); + resolve(r); + }); + helper.stdin.write(`${JSON.stringify({ ...fields, id, cmd })}\n`); + }); + +let n = 0; +for (const step of JSON.parse(readFileSync(commandsPath, 'utf8'))) { + if (typeof step === 'string') { + log(`run ${step}`); + spawn(step, [], { detached: true, stdio: 'ignore', shell: true }).unref(); + continue; + } + if (step.sleep) { + await new Promise((r) => setTimeout(r, step.sleep)); + continue; + } + const { cmd, ...fields } = step; + const started = Date.now(); + const reply = await call(cmd, fields); + n++; + const { screenshot, text, ...rest } = reply; + log(`== ${n} ${cmd} ${JSON.stringify(fields)} (${Date.now() - started} ms) ${JSON.stringify(rest)}`); + if (text) log(text.split('\n').slice(0, 80).join('\n')); + if (screenshot) { + const file = join(out, `${n}-${cmd}.jpg`); + writeFileSync(file, Buffer.from(screenshot.jpegBase64, 'base64')); + log(` picture ${screenshot.width}x${screenshot.height} → ${file}`); + } +} +await call('stop', {}); diff --git a/scripts/gnome-headless.sh b/scripts/gnome-headless.sh new file mode 100755 index 00000000..caab16b1 --- /dev/null +++ b/scripts/gnome-headless.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Runs a command inside a private, headless GNOME Shell that has Stem's +# computer-control extension loaded (native/linux/gnome-extension, from the +# tree this script sits in) and a working accessibility bus — the setup +# computer control needs, without touching the person's own session: +# +# - its own D-Bus session (dbus-run-session), runtime, config, data and cache +# folders under /tmp/stem-gnome; +# - GSETTINGS_BACKEND=keyfile, so no setting is ever written to the person's +# dconf database (the private bus would otherwise reach their real one); +# - everything it started is stopped when the command ends. +# +# Used by `scripts/linux-pc.mjs gnome-e2e|gnome-sh`. Inside, WAYLAND_DISPLAY +# points at the shell and XDG_CURRENT_DESKTOP=GNOME, so the Linux helper and +# Electron behave as in a real GNOME Wayland session. +set -u +root=/tmp/stem-gnome +repo=$(cd "$(dirname "$0")/.." && pwd) +if [ -z "${STEM_GNOME_INNER:-}" ]; then + rm -rf "$root" + mkdir -p "$root/config" "$root/data/gnome-shell/extensions" "$root/cache" "$root/run" + chmod 700 "$root/run" + ln -s "$repo/native/linux/gnome-extension/stem-computer@awantech.sk" "$root/data/gnome-shell/extensions/" + export XDG_CONFIG_HOME=$root/config XDG_DATA_HOME=$root/data XDG_CACHE_HOME=$root/cache XDG_RUNTIME_DIR=$root/run + export GSETTINGS_BACKEND=keyfile STEM_GNOME_INNER=1 + unset DISPLAY WAYLAND_DISPLAY DBUS_SESSION_BUS_ADDRESS + exec dbus-run-session -- "$0" "$@" +fi +[ "$GSETTINGS_BACKEND" = keyfile ] || { echo "gnome-headless: refusing to run off the keyfile settings backend" >&2; exit 1; } +gsettings set org.gnome.shell enabled-extensions "['stem-computer@awantech.sk']" +gsettings set org.gnome.shell disable-user-extensions false +# D-Bus activation cannot start the accessibility daemons inside a +# dbus-run-session here (exec is denied), so they are started by hand. +/usr/libexec/at-spi-bus-launcher --launch-immediately >"$root/a11y.log" 2>&1 & +for _ in $(seq 50); do gdbus call --session --dest org.a11y.Bus --object-path /org/a11y/bus --method org.a11y.Bus.GetAddress >/dev/null 2>&1 && break; sleep 0.1; done +/usr/libexec/at-spi2-registryd >>"$root/a11y.log" 2>&1 & +gnome-shell --headless --wayland --no-x11 --virtual-monitor 1440x900 --wayland-display=stem-gnome >"$root/shell.log" 2>&1 & +for _ in $(seq 100); do + gdbus call --session --dest org.gnome.Shell --object-path /sk/awantech/StemComputer --method sk.awantech.StemComputer.Version >/dev/null 2>&1 && break + sleep 0.2 +done +export WAYLAND_DISPLAY=stem-gnome XDG_SESSION_TYPE=wayland XDG_CURRENT_DESKTOP=GNOME +"$@" +status=$? +for p in $(pgrep -u "$(id -u)"); do + [ "$p" = "$$" ] && continue + tr '\0' '\n' <"/proc/$p/environ" 2>/dev/null | grep -q "^XDG_CONFIG_HOME=$root/config\$" && kill "$p" 2>/dev/null +done 2>/dev/null +exit $status diff --git a/scripts/linux-pc.mjs b/scripts/linux-pc.mjs new file mode 100644 index 00000000..1ec5ee3d --- /dev/null +++ b/scripts/linux-pc.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node +// Run Stem on a real Linux PC over SSH: the working tree is copied over, so +// uncommitted work is what gets tested, the way scripts/linux-test.mjs does it +// with a container. The PC needs Node 24, rsync and GNOME (mutter); the host +// comes from STEM_LINUX_HOST, e.g. `STEM_LINUX_HOST=join3r@192.168.1.164`. +// +// node scripts/linux-pc.mjs test [vitest args] unit tests +// node scripts/linux-pc.mjs e2e [playwright args] the E2E suite on a headless Wayland desktop +// node scripts/linux-pc.mjs desktop Stem on that headless desktop, a fresh profile; +// its DevTools port is forwarded to localhost:9333 +// node scripts/linux-pc.mjs screen Stem on the PC's own screen, DevTools on localhost:9334 +// node scripts/linux-pc.mjs gnome-e2e [playwright args] E2E specs inside a private headless GNOME Shell with +// Stem's extension loaded (computer control, e.g. +// STEM_E2E_COMPUTER_APP=zenity tests/e2e/computer-live.spec.ts) +// node scripts/linux-pc.mjs gnome-sh '' one command in that GNOME session +// node scripts/linux-pc.mjs shell a shell in the copy +// node scripts/linux-pc.mjs sh '' one command in the copy +// node scripts/linux-pc.mjs sync only copy and install +// +// The headless desktop is a private mutter with its own D-Bus session, so +// nothing appears on the PC's screen and the person at it is not disturbed. It +// is real Wayland on the real GPU, unlike the container's Xvfb. + +import { spawnSync } from 'node:child_process'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const host = process.env.STEM_LINUX_HOST; +if (!host) { + console.error('linux-pc: set STEM_LINUX_HOST, e.g. STEM_LINUX_HOST=join3r@192.168.1.164'); + process.exit(2); +} +const DIR = 'stem'; +const SSH = ['-o', 'BatchMode=yes', '-o', 'VisualHostKey=no']; +const [mode = 'test', ...rest] = process.argv.slice(2); + +function run(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { stdio: 'inherit', ...opts }); + if (r.status !== 0) process.exit(r.status ?? 1); +} +const quote = (s) => `'${String(s).replace(/'/g, `'\\''`)}'`; + +run('rsync', [ + '-az', '--delete', + '--filter=:- .gitignore', '--exclude=.git', '--exclude=.worktrees', + // node_modules and dist are the PC's own (Linux native modules), never the Mac's. + '--filter=P node_modules/', '--filter=P dist/', '--filter=P out/', + '-e', `ssh ${SSH.join(' ')}`, + `${repo}/`, `${host}:${DIR}/` +]); + +// Runs on the PC. `headless` starts a private mutter on a virtual monitor and +// points Wayland apps at it; the trap takes it down with the session. +const prelude = ` +set -euo pipefail +export PATH="$HOME/.local/bin:$PATH" XDG_RUNTIME_DIR=/run/user/$(id -u) +cd ~/${DIR} +want=$(sha256sum package-lock.json | cut -d' ' -f1) +if [ "$(cat node_modules/.stem-lock 2>/dev/null)" != "$want" ]; then + echo "linux-pc: npm ci (lockfile changed)" + npm ci --no-audit --no-fund + echo "$want" > node_modules/.stem-lock +fi +headless() { + local name=stem-pc-$$ + mutter --headless --wayland --no-x11 --virtual-monitor 1440x900 --wayland-display=$name >/tmp/$name.log 2>&1 & + local pid=$! + trap "kill $pid 2>/dev/null" EXIT + for _ in $(seq 50); do [ -S "$XDG_RUNTIME_DIR/$name" ] && break; sleep 0.1; done + export WAYLAND_DISPLAY=$name XDG_SESSION_TYPE=wayland + unset DISPLAY +} +export -f headless +# The private D-Bus session chatters on stderr; it goes to a log and the +# command's own stderr comes back through fd 3. +exec 3>&2 +`; +const args = rest.map(quote).join(' '); +const body = { + sync: 'true', + test: `exec npx vitest run ${args}`, + e2e: `exec dbus-run-session -- bash -c ${quote(`exec 2>&3; headless; npx playwright test ${args}`)} 2>/tmp/stem-pc-dbus.log`, + desktop: `npx electron-vite build >/tmp/stem-pc-build.log +echo "linux-pc: Stem on a headless Wayland desktop; DevTools at localhost:9333" +exec dbus-run-session -- bash -c ${quote('exec 2>&3; headless; npx electron . --fresh --remote-debugging-port=9333')} 2>/tmp/stem-pc-dbus.log`, + screen: `npx electron-vite build >/tmp/stem-pc-build.log +export WAYLAND_DISPLAY=wayland-0 XDG_SESSION_TYPE=wayland DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus +echo "linux-pc: Stem on the PC's screen; DevTools at localhost:9334" +exec npx electron . --fresh --remote-debugging-port=9334`, + // A private GNOME Shell (scripts/gnome-headless.sh), never the person's session. + 'gnome-e2e': `exec ./scripts/gnome-headless.sh bash -c ${quote(`exec 2>&3; ${process.env.STEM_E2E_COMPUTER_APP ? `export STEM_E2E_COMPUTER_APP=${quote(process.env.STEM_E2E_COMPUTER_APP)};` : ''} npx playwright test ${args}`)} 2>/tmp/stem-pc-gnome.log`, + 'gnome-sh': `exec ./scripts/gnome-headless.sh bash -c ${quote(rest.join(' '))}`, + shell: 'exec bash -l', + sh: rest.join(' ') +}[mode]; +if (!body) { + console.error('usage: linux-pc.mjs test|e2e|desktop|screen|gnome-e2e|gnome-sh|shell|sync|sh [args]'); + process.exit(2); +} + +const forward = { desktop: ['-L', '9333:localhost:9333'], screen: ['-L', '9334:localhost:9334'] }[mode] ?? []; +const tty = mode === 'shell' && process.stdin.isTTY ? ['-t'] : []; +run('ssh', [...SSH, ...tty, ...forward, host, `bash -c ${quote(prelude + body)}`]); diff --git a/scripts/linux-test.mjs b/scripts/linux-test.mjs new file mode 100644 index 00000000..e4c8f215 --- /dev/null +++ b/scripts/linux-test.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node +// Run Stem on Linux from any machine with Docker (a Mac included), in the +// container tests/linux/Dockerfile describes. +// +// node scripts/linux-test.mjs [test] [playwright args] the E2E suite under Xvfb +// node scripts/linux-test.mjs desktop Stem on a Linux desktop: +// open http://localhost:6080/vnc.html +// node scripts/linux-test.mjs shell a shell in the same container +// +// e.g. `node scripts/linux-test.mjs test tests/e2e/browser-live.spec.ts`. +// +// The repo is mounted as it is on disk, so uncommitted work is what gets +// tested. node_modules, dist and the browsers live in Docker volumes (Linux +// builds of the native modules); the first run spends a few minutes on +// `npm ci`, later runs only when package-lock.json changed. +// +// On this Mac, a docker command that just sits there is the locked login +// keychain, not the network (see the Docker notes in the memory). + +import { spawnSync } from 'node:child_process'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const IMAGE = 'stem-linux-test'; +const [mode = 'test', ...rest] = process.argv.slice(2); +const known = ['test', 'desktop', 'shell']; +const cmd = known.includes(mode) ? [mode, ...rest] : ['test', mode, ...rest]; + +function run(args, opts = {}) { + const r = spawnSync('docker', args, { stdio: 'inherit', ...opts }); + if (r.status !== 0) process.exit(r.status ?? 1); +} + +run(['build', '-q', '-t', IMAGE, join(repo, 'tests/linux')], { stdio: ['ignore', 'ignore', 'inherit'] }); + +const interactive = cmd[0] === 'shell' || cmd[0] === 'desktop'; +run([ + 'run', + '--rm', + '--init', + ...(interactive && process.stdin.isTTY ? ['-it'] : []), + ...(cmd[0] === 'desktop' ? ['--name', 'stem-linux-desktop', '-p', '127.0.0.1:6080:6080'] : []), + // Electron's and Chromium's sandboxes need user namespaces, which Docker's + // default seccomp profile refuses. A test container, not a place to keep secrets. + '--security-opt', 'seccomp=unconfined', + '--shm-size', '1g', + '-v', `${repo}:/repo`, + '-v', 'stem-linux-node-modules:/repo/node_modules', + '-v', 'stem-linux-dist:/repo/dist', + '-v', 'stem-linux-playwright:/home/node/.cache/ms-playwright', + '-v', 'stem-linux-npm:/home/node/.npm', + '-e', 'STEM_E2E_BROWSER=/usr/bin/chromium', + '-e', 'CI=1', + IMAGE, + ...cmd +]); diff --git a/scripts/notarize-mac-dmg.sh b/scripts/notarize-mac-dmg.sh index 71fae816..7103d8db 100644 --- a/scripts/notarize-mac-dmg.sh +++ b/scripts/notarize-mac-dmg.sh @@ -3,6 +3,7 @@ set -euo pipefail verify_only=false +dmg_stapled=true if [ "${1:-}" = --verify-only ]; then verify_only=true shift @@ -32,6 +33,18 @@ cleanup() { } trap cleanup EXIT +# stapler asks Apple's CloudKit for the ticket, and that connection drops now +# and then from CI runners (Error 68) though nothing is wrong with the file. +staple_retry() { + local attempt + for attempt in 1 2 3 4 5 6; do + if xcrun stapler "$@"; then return 0; fi + [ "$attempt" = 6 ] && return 1 + echo "stapler $1 failed (attempt $attempt); retrying in 20s" >&2 + sleep 20 + done +} + if [ "$verify_only" = false ]; then : "${APPLE_API_KEY:?Notarization key path is required}" : "${APPLE_API_KEY_ID:?Notarization key ID is required}" @@ -47,10 +60,17 @@ if [ "$verify_only" = false ]; then process.exit(1); } ' "$work_dir/notarization.json" - xcrun stapler staple "$dmg" + # Apple has accepted the DMG at this point, and the app inside carries its own + # stapled ticket (checked below). A ticket stapled to the DMG only spares + # Gatekeeper an online lookup on first open, so a CloudKit outage on the + # runner is a warning here, not a failed release. + if ! staple_retry staple "$dmg"; then + echo "::warning::Apple accepted the DMG but its ticket could not be stapled; Gatekeeper will fetch it online on first open." + dmg_stapled=false + fi fi -xcrun stapler validate "$dmg" +if [ "$dmg_stapled" = true ]; then staple_retry validate "$dmg"; fi codesign --verify --strict --verbose=2 -R "$requirement" "$dmg" spctl --assess --type open --context context:primary-signature --verbose=4 "$dmg" @@ -60,7 +80,7 @@ mounted=true app="$mount_point/Stem.app" test "$(lipo -archs "$app/Contents/MacOS/Stem")" = arm64 codesign --verify --deep --strict --verbose=2 -R "$requirement" "$app" -xcrun stapler validate "$app" +staple_retry validate "$app" spctl --assess --type execute --verbose=4 "$app" syspolicy_check distribution "$app" hdiutil detach "$mount_point" diff --git a/scripts/windows-vm.mjs b/scripts/windows-vm.mjs new file mode 100644 index 00000000..34c259b3 --- /dev/null +++ b/scripts/windows-vm.mjs @@ -0,0 +1,120 @@ +#!/usr/bin/env node +// Run Stem on a Windows machine over SSH, the way scripts/linux-pc.mjs does on +// Linux: the working tree (uncommitted work included) is copied over and the +// command runs there. The machine needs OpenSSH with PowerShell as its shell, +// Node 24 on the user PATH, and Git for Windows. +// +// STEM_WINDOWS_HOST user@host[:port], e.g. stem@127.0.0.1:2222 +// STEM_WINDOWS_JUMP optional jump host, e.g. join3r@192.168.1.164 (a VM +// whose ports are bound to that machine's loopback) +// +// node scripts/windows-vm.mjs test [vitest args] unit tests +// node scripts/windows-vm.mjs e2e [playwright args] the E2E suite +// node scripts/windows-vm.mjs shell PowerShell in the copy +// node scripts/windows-vm.mjs ps '' one PowerShell command there +// node scripts/windows-vm.mjs desktop '' the same, inside the logged-in desktop +// session (real input, capture, UI Automation) +// node scripts/windows-vm.mjs sync only copy and install +// +// Commands run in the SSH session, which has no interactive desktop: windows +// open and render, but nothing shows on the VM's screen, and input, screen +// capture and UI Automation of other apps do not work there. `desktop` runs +// its command in the console session instead, through an interactive +// scheduled task, and prints what it wrote; the user must be logged in. + +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const target = process.env.STEM_WINDOWS_HOST; +if (!target) { + console.error('windows-vm: set STEM_WINDOWS_HOST (user@host[:port]), and STEM_WINDOWS_JUMP if it sits behind one'); + process.exit(2); +} +const [userHost, port = '22'] = target.split(/:(?=\d+$)/); +const jump = process.env.STEM_WINDOWS_JUMP; +const opts = [ + '-o', 'BatchMode=yes', '-o', 'VisualHostKey=no', '-o', 'StrictHostKeyChecking=accept-new', + ...(jump ? ['-o', `ProxyCommand=ssh -o BatchMode=yes -o VisualHostKey=no -W %h:%p ${jump}`] : []) +]; +const [mode = 'test', ...rest] = process.argv.slice(2); + +function run(cmd, args, o = {}) { + const r = spawnSync(cmd, args, { stdio: 'inherit', ...o }); + if (r.status !== 0) process.exit(r.status ?? 1); +} +const psQuote = (s) => `'${String(s).replace(/'/g, "''")}'`; + +// The tracked and untracked-but-not-ignored files, as one tarball; Windows' +// own tar.exe unpacks it. +const tmp = mkdtempSync(join(tmpdir(), 'stem-win-')); +const tarball = join(tmp, 'stem.tgz'); +const files = spawnSync('git', ['ls-files', '-co', '--exclude-standard', '-z'], { cwd: repo }); +if (files.status !== 0) process.exit(1); +run('tar', ['--null', '-T', '-', '-czf', tarball], { cwd: repo, input: files.stdout, stdio: ['pipe', 'inherit', 'inherit'], env: { ...process.env, COPYFILE_DISABLE: '1' } }); +run('scp', ['-q', ...opts, '-P', port, tarball, `${userHost}:stem.tgz`]); +rmSync(tmp, { recursive: true, force: true }); + +// Runs on Windows. node_modules (the Windows builds of the native modules), +// dist and release (the last installer build) survive; everything else is +// replaced by the copy. +const prelude = ` +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +$env:Path = [Environment]::GetEnvironmentVariable('Path','User') + ';' + [Environment]::GetEnvironmentVariable('Path','Machine') +$dst = "$env:USERPROFILE\\stem" +New-Item -ItemType Directory -Force $dst | Out-Null +Get-ChildItem -Force $dst | Where-Object { $_.Name -notin 'node_modules','dist','release' } | Remove-Item -Recurse -Force +tar -xzf "$env:USERPROFILE\\stem.tgz" -C $dst +Remove-Item "$env:USERPROFILE\\stem.tgz" +Set-Location $dst +$want = (Get-FileHash package-lock.json -Algorithm SHA256).Hash +$have = if (Test-Path node_modules\\.stem-lock) { Get-Content node_modules\\.stem-lock } else { '' } +if ($have -ne $want) { + Write-Output 'windows-vm: npm ci (lockfile changed)' + cmd /c 'npm ci --no-audit --no-fund 2>&1' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Set-Content node_modules\\.stem-lock $want +} +$ErrorActionPreference = 'Continue' +`; +const args = rest.map(psQuote).join(' '); +const body = { + sync: '', + test: `& npx.cmd vitest run ${args}; exit $LASTEXITCODE`, + e2e: `& npx.cmd playwright test ${args}; exit $LASTEXITCODE`, + shell: '', + ps: `${rest.join(' ')}\nexit $LASTEXITCODE`, + desktop: ` +$log = "$env:TEMP\\stem-desktop.log" +Remove-Item "$log", "$log.done" -ErrorAction SilentlyContinue +$script = "$env:TEMP\\stem-desktop.ps1" +Set-Content $script ("Set-Location '$dst'" + [Environment]::NewLine + "$env:Path = '$env:Path'" + [Environment]::NewLine + "& { " + [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${Buffer.from(rest.join(' ')).toString('base64')}')) + " } *> '$log'" + [Environment]::NewLine + "Set-Content '$log.done' $LASTEXITCODE") +$action = New-ScheduledTaskAction -Execute powershell.exe -Argument "-NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File \`"$script\`"" +$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive +Register-ScheduledTask -TaskName StemDesktop -Action $action -Principal $principal -Force | Out-Null +Start-ScheduledTask -TaskName StemDesktop +$deadline = (Get-Date).AddMinutes(20) +while (-not (Test-Path "$log.done") -and (Get-Date) -lt $deadline) { Start-Sleep 1 } +Get-Content $log -ErrorAction SilentlyContinue +$code = Get-Content "$log.done" -ErrorAction SilentlyContinue +exit [int]("0" + $code)` +}[mode]; +if (body === undefined) { + console.error('usage: windows-vm.mjs test|e2e|shell|sync|ps|desktop [args]'); + process.exit(2); +} + +const ssh = ['-p', port, ...opts, userHost]; +// -EncodedCommand, not a script on stdin: `powershell -Command -` stops at the +// first block that spans lines. +const encoded = (script) => `powershell -NoProfile -NonInteractive -EncodedCommand ${Buffer.from(script, 'utf16le').toString('base64')}`; +if (mode === 'shell') { + run('ssh', [...ssh, encoded(prelude)]); + run('ssh', ['-t', ...ssh, 'cd $env:USERPROFILE\\stem; powershell -NoProfile']); +} else { + run('ssh', [...ssh, encoded(prelude + body)]); +} diff --git a/src/browser-extension/background.js b/src/browser-extension/background.js index b2da3ee1..60c3a3a3 100644 --- a/src/browser-extension/background.js +++ b/src/browser-extension/background.js @@ -1,4 +1,4 @@ -/* global chrome, navigator, setTimeout, clearTimeout */ +/* global chrome, crypto, navigator, setTimeout, clearTimeout, URL */ import { initActions, notifyStopped, perform } from './actions.js'; import * as cdp from './cdp.js'; import { noteDownloadStart, trackDownloads } from './downloads.js'; @@ -33,6 +33,50 @@ let attempt = 0; let reconnectTimer = null; let hostConnected = null; let lastError = ''; +/** What Stem calls this browser profile ("Work"), once it has said. */ +let profileName = ''; + +// ---- this profile ---- + +const PROFILE_KEY = 'stemProfileId'; + +/** + * A random id kept in this extension's storage — one per browser profile, as + * each profile runs its own copy of the extension. It is how Stem tells Arc's + * work profile from its personal one; it says nothing else about the profile. + */ +async function profileId() { + const got = await chrome.storage.local.get(PROFILE_KEY).catch(() => ({})); + if (typeof got[PROFILE_KEY] === 'string' && got[PROFILE_KEY]) return got[PROFILE_KEY]; + const id = crypto.randomUUID(); + await chrome.storage.local.set({ [PROFILE_KEY]: id }).catch(() => {}); + return id; +} + +/** The three sites most open here ("mail.google.com, github.com"), so the user can tell unnamed profiles apart. */ +async function sitesHint() { + try { + const counts = new Map(); + for (const t of await chrome.tabs.query({})) { + if (t.incognito) continue; + let host = ''; + try { + const u = new URL(t.url || t.pendingUrl || ''); + if (u.protocol === 'http:' || u.protocol === 'https:') host = u.hostname; + } catch { + // Not a URL (an empty new tab). + } + if (host) counts.set(host, (counts.get(host) || 0) + 1); + } + return [...counts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 3) + .map(([host]) => host) + .join(', '); + } catch { + return ''; + } +} // ---- native port ---- @@ -46,9 +90,20 @@ function post(message) { } } -function connect() { - if (port) return; +let connecting = false; + +async function connect() { + if (port || connecting) return; clearTimeout(reconnectTimer); + connecting = true; + let profile; + let hint; + try { + [profile, hint] = await Promise.all([profileId(), sitesHint()]); + } finally { + connecting = false; + } + if (port) return; let p; try { p = chrome.runtime.connectNative(NATIVE_HOST); @@ -76,7 +131,9 @@ function connect() { protocol: PROTOCOL, extensionId: chrome.runtime.id, extensionVersion: chrome.runtime.getManifest().version, - userAgent: navigator.userAgent + userAgent: navigator.userAgent, + profileId: profile, + ...(hint ? { hint } : {}) }); refreshBadge(); } @@ -84,7 +141,7 @@ function connect() { function scheduleReconnect() { clearTimeout(reconnectTimer); if (attempt < RECONNECT_STEPS_MS.length) { - reconnectTimer = setTimeout(connect, RECONNECT_STEPS_MS[attempt++]); + reconnectTimer = setTimeout(() => void connect(), RECONNECT_STEPS_MS[attempt++]); } // Past the quick steps, the 30-second alarm keeps trying. } @@ -143,6 +200,9 @@ async function onNativeMessage(msg) { hostConnected = !!msg.connected; refreshBadge(); return; + case 'profile': + profileName = typeof msg.name === 'string' ? msg.name.slice(0, 60) : ''; + return; default: } } @@ -249,6 +309,7 @@ async function popupState() { port: !!port, host: hostConnected, lastError: port ? '' : lastError, + profile: port && hostConnected !== false ? profileName : '', runs: list, version: chrome.runtime.getManifest().version }; @@ -269,8 +330,7 @@ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => { } if (msg.type === 'stem:reconnect') { attempt = 0; - connect(); - void popupState().then(sendResponse); + void connect().then(() => popupState()).then(sendResponse); return true; } return false; @@ -280,7 +340,7 @@ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => { chrome.alarms.onAlarm.addListener((alarm) => { if (alarm.name !== ALARM) return; - if (!port) connect(); + if (!port) void connect(); const now = Date.now(); for (const r of runs.live()) { if (now - r.lastActionAt > RUN_IDLE_MS) void endRun(r.threadId); @@ -289,7 +349,7 @@ chrome.alarms.onAlarm.addListener((alarm) => { function start() { chrome.alarms.create(ALARM, { periodInMinutes: 0.5 }).catch(() => {}); - connect(); + void connect(); } chrome.runtime.onStartup.addListener(start); diff --git a/src/browser-extension/cdp.js b/src/browser-extension/cdp.js index ef0ba347..25c0ed52 100644 --- a/src/browser-extension/cdp.js +++ b/src/browser-extension/cdp.js @@ -1,4 +1,4 @@ -/* global chrome, setTimeout, clearTimeout, setInterval */ +/* global chrome, setTimeout, clearTimeout, setInterval, document, HTMLIFrameElement, HTMLEmbedElement, HTMLObjectElement */ import { ConsoleLog, NetworkLog } from './logs.js'; import { installMarkerHooks, markerCall, MARKER_BINDING, MARKER_PING_MS } from './marker.js'; import { UserError } from './util.js'; @@ -23,6 +23,15 @@ export const IDLE_DETACH_MS = 3 * 60_000; const sessions = new Map(); const attaching = new Map(); +/** + * The snapshot of a tab Chrome detached Stem from on its own (another + * extension's frame appearing — a password manager's fill menu opening on the + * field Stem just focused), for the re-attach to take back when the document + * is still the same: node ids outlive the debugger session, so the model's + * refs keep working. + */ +const orphanSnapshots = new Map(); +const ORPHAN_SNAPSHOT_MS = 2 * 60_000; const queues = new Map(); const handlers = { markerStop: () => {}, detached: () => {}, downloadStart: () => {} }; @@ -110,12 +119,24 @@ export function attachedTabs() { return [...sessions.keys()]; } +/** chrome.debugger's refusal when a page holds another extension's frame. */ +const FOREIGN_FRAME = /chrome-extension:\/\/ URL of different extension/i; + function attachError(tabId, message) { if (/another debugger/i.test(message)) { return new UserError( `Something else is already debugging tab ${tabId} (another extension, or remote debugging): Stem can't work in it until that lets go.` ); } + // A normal page that holds another extension's frame — a password manager's + // fill menu, say — refuses the debugger as a whole (2026-10-19: Bitwarden in + // a Cloudflare billing tab). + if (FOREIGN_FRAME.test(message)) { + return new UserError( + `Another extension has a frame inside tab ${tabId} (often a password manager's fill menu), and the browser then lets no other extension control the tab. ` + + 'Open the same address in a new tab and work there, and tell the user that is where the result is; or ask them to close that menu and try again.' + ); + } if (/cannot (access|attach)|chrome:\/\/|chrome-extension|webstore|not allowed/i.test(message)) { return new UserError(`The browser does not let extensions control tab ${tabId} (${message}).`); } @@ -133,11 +154,55 @@ export function attach(tabId) { return p; } +/** + * Take out the frames other extensions put in a page (in every frame of the + * tab, closed shadow roots included), so the debugger may attach. Answers how + * many went; 0 when none were found or scripting is not allowed there. + */ +async function removeForeignFrames(tabId) { + try { + const results = await chrome.scripting.executeScript({ + target: { tabId, allFrames: true }, + args: [chrome.runtime.id], + func: (ownId) => { + const own = `chrome-extension://${ownId}/`; + let removed = 0; + const visit = (root) => { + for (const el of root.querySelectorAll('*')) { + const src = el instanceof HTMLIFrameElement || el instanceof HTMLEmbedElement ? el.src : el instanceof HTMLObjectElement ? el.data : ''; + if (/^chrome-extension:\/\//i.test(src || '') && !src.startsWith(own)) { + el.remove(); + removed++; + continue; + } + const shadow = chrome.dom?.openOrClosedShadowRoot?.(el) ?? el.shadowRoot; + if (shadow) visit(shadow); + } + }; + visit(document); + return removed; + } + }); + return results.reduce((n, r) => n + (typeof r.result === 'number' ? r.result : 0), 0); + } catch { + return 0; + } +} + async function doAttach(tabId) { try { await chrome.debugger.attach({ tabId }, '1.3'); } catch (err) { - throw attachError(tabId, cdpMessage(err)); + const message = cdpMessage(err); + // Another extension's frame (a password manager's fill menu) blocks the + // debugger for the whole tab. It is that extension's overlay, not the + // page: remove it and try once more. + if (!FOREIGN_FRAME.test(message) || !(await removeForeignFrames(tabId))) throw attachError(tabId, message); + try { + await chrome.debugger.attach({ tabId }, '1.3'); + } catch (again) { + throw attachError(tabId, cdpMessage(again)); + } } const s = new Session(tabId); sessions.set(tabId, s); @@ -153,6 +218,9 @@ async function doAttach(tabId) { const { frameTree } = await s.send('Page.getFrameTree'); s.mainFrameId = frameTree.frame.id; s.loaderId = frameTree.frame.loaderId; + const orphan = orphanSnapshots.get(tabId); + orphanSnapshots.delete(tabId); + if (orphan && orphan.snapshot.loaderId === s.loaderId && Date.now() - orphan.at < ORPHAN_SNAPSHOT_MS) s.snapshot = orphan.snapshot; await installMarkerHooks((m, p) => s.send(m, p)); await ping(s); } catch (err) { @@ -302,6 +370,7 @@ chrome.debugger.onDetach.addListener((source, reason) => { if (s) { sessions.delete(tabId); clearTimeout(s.idleTimer); + if (s.snapshot && reason !== 'canceled_by_user') orphanSnapshots.set(tabId, { snapshot: s.snapshot, at: Date.now() }); } handlers.detached(tabId, reason); }); diff --git a/src/browser-extension/manifest.json b/src/browser-extension/manifest.json index 9d26fdd8..b5611063 100644 --- a/src/browser-extension/manifest.json +++ b/src/browser-extension/manifest.json @@ -12,8 +12,10 @@ "nativeMessaging", "storage", "alarms", - "webNavigation" + "webNavigation", + "scripting" ], + "host_permissions": [""], "background": { "service_worker": "background.js", "type": "module" diff --git a/src/browser-extension/popup.js b/src/browser-extension/popup.js index 39668b01..b6e9bda1 100644 --- a/src/browser-extension/popup.js +++ b/src/browser-extension/popup.js @@ -26,7 +26,7 @@ function render(state) { text = 'Stem app isn’t running'; } else { tone = 'on'; - text = 'Connected to Stem'; + text = state.profile ? `Connected to Stem as “${state.profile}”` : 'Connected to Stem'; } status.dataset.tone = tone; $('status-text').textContent = text; diff --git a/src/desktop/app-menu.ts b/src/desktop/app-menu.ts index d97c9fe0..13295a3f 100644 --- a/src/desktop/app-menu.ts +++ b/src/desktop/app-menu.ts @@ -1,48 +1,279 @@ import { Menu, type MenuItemConstructorOptions } from 'electron'; import { isMac } from './platform'; +import { SHORTCUTS, chordFor, type Chord, type ShortcutId } from '../shared/shortcut-defs'; +import type { ManageTabId, MenuCommand } from '../shared/types'; -// Stem's application menu: Electron's stock menu, minus "Close Window". +// Stem's application menu. // -// Left alone, Electron installs a default menu whose File menu holds exactly one -// item — Close Window, bound to ⌘W (Ctrl+W off macOS). That binding is a browser -// reflex, and Stem is not a browser: there is one main window, it is the app, and -// closing it destroys the renderer (the open chat, the scroll position, the draft -// in the composer) while the process keeps running behind the dock icon. A ⌘W -// aimed at the editor or the browser next door lands here as "where did Stem go?". +// What is NOT in it: Close Window. Left alone, Electron installs a default menu +// whose File menu holds Close Window, bound to ⌘W (Ctrl+W off macOS). That +// binding is a browser reflex, and Stem is not a browser: there is one main +// window, it is the app, and closing it destroys the renderer (the open chat, the +// scroll position, the draft in the composer) while the process keeps running +// behind the dock icon. A ⌘W aimed at the editor or the browser next door lands +// here as "where did Stem go?". The accelerator is also window-agnostic: pressed +// while the Quick Chat overlay has focus it closed THAT window, and the overlay is +// constructed once at startup and thereafter only ever shown and hidden (see +// quickchat/windows.ts) — so a stray ⌘W left Quick Chat dead until restart. +// Hence: no close item anywhere, and therefore no ⌘W. ⌘Q / Ctrl+Q still quits, +// and the red traffic light (the native close button off macOS) still closes the +// main window. // -// The accelerator is also window-agnostic, which makes it worse than an annoyance: -// pressed while the Quick Chat overlay has focus it closes THAT window, and the -// overlay is constructed once at startup and thereafter only ever shown and hidden -// (see quickchat/windows.ts) — so a stray ⌘W left Quick Chat dead until restart. +// What IS in it: every action that has a shortcut, so the menu bar is where a +// person discovers them. The keyboard stays the renderer's job — the menu never +// owns a key, it only prints one: // -// Hence: no close item anywhere, and therefore no ⌘W. Everything else is the stock -// menu, spelled out with roles so Electron keeps supplying the platform's own -// labels, translations and behavior — Edit in particular *must* keep its roles or -// copy/paste stop working on macOS. +// - Off macOS every Stem item carries `registerAccelerator: false`, so the label +// shows the keycap and the renderer's keydown listener does the work. +// - On macOS that flag does not exist, but Electron hands a key to the page +// first and offers it to the menu only when the page left it unhandled (no +// preventDefault). The shortcut listener calls preventDefault whenever it ran a +// handler, so a key reaches the menu only when nothing on screen claimed it — +// and the menu then sends the same command, which still finds no handler. // -// What this does not change: ⌘Q / Ctrl+Q still quits, the red traffic light (and -// the native close button off macOS) still closes the main window, and the way -// back in is unchanged — the dock, the tray, or a plain `stem` (see the -// window-all-closed note in index.ts). - -/** The menu template for a platform. Pure data, so it can be asserted in a unit test. */ -export function appMenuTemplate(mac: boolean): MenuItemConstructorOptions[] { - if (mac) { - // No File menu: Close Window was its only item. macOS's `windowMenu` role is - // already close-free (Minimize, Zoom, Bring All to Front), so it stays as-is. - return [{ role: 'appMenu' }, { role: 'editMenu' }, { role: 'viewMenu' }, { role: 'windowMenu' }]; - } - // Off macOS the split differs: File holds Quit (keep it) and the `windowMenu` - // role is the one carrying Close, so that menu is spelled out without it. - return [ - { role: 'fileMenu' }, - { role: 'editMenu' }, - { role: 'viewMenu' }, - { label: 'Window', submenu: [{ role: 'minimize' }] } - ]; -} - -/** Replace Electron's default menu with Stem's. Call once, after the app is ready. */ -export function installAppMenu(): void { - Menu.setApplicationMenu(Menu.buildFromTemplate(appMenuTemplate(isMac))); +// Either way a click arrives as a MenuCommand on the main window, and a +// `shortcut` command runs the handler the key would have. The roles (Edit in +// particular) stay roles, so Electron keeps supplying the platform's own labels +// and behavior — copy/paste stop working on macOS without them. + +/** A chat the Window menu lists under ⌘1–9. */ +export interface MenuChat { + threadId: string; + title: string; +} + +/** What the template is built from — everything that can change while Stem runs. */ +export interface AppMenuState { + /** The rows ⌘1–9 open right now, top first; empty while the numbers are not live. */ + chats: MenuChat[]; + /** The Quick Chat summon key, when one is bound; printed beside the item. */ + quickChatAccelerator: string | null; + /** False where this install has no update path (dev builds, e2e). */ + canCheckForUpdates: boolean; +} + +export interface AppMenuActions { + /** Deliver a command to the main window. */ + command(command: MenuCommand): void; + toggleQuickChat(): void; + checkForUpdates(): void; +} + +/** + * Commands that act on what is in front of the user — the open chat's composer, + * the selected rows. Picked while another window (the Quick Chat overlay) has + * focus they would reach into a window the user is not looking at, so the caller + * drops them there instead of revealing the main window. + */ +export const IN_WINDOW_SHORTCUTS: ReadonlySet = new Set([ + 'cycle-effort', + 'toggle-speed', + 'toggle-format', + 'attach', + 'stop', + 'archive-thread', + 'snooze-thread', + 'toggle-read', + 'delete-thread' +]); + +/** Electron accelerator for a chord, e.g. 'CmdOrCtrl+Shift+N', 'Ctrl+X'. */ +export function chordAccelerator(chord: Chord, key: string = chord.key): string { + const parts: string[] = []; + if (chord.mod) parts.push('CmdOrCtrl'); + if (chord.control) parts.push('Ctrl'); + if (chord.shift) parts.push('Shift'); + parts.push(key); + return parts.join('+'); +} + +const SMALL_WORDS = new Set(['a', 'an', 'and', 'or', 'the', 'of', 'to']); + +/** 'Mark read or unread' → 'Mark Read or Unread': menus are title case. */ +export function titleCase(label: string): string { + return label + .split(' ') + .map((w, i) => (i > 0 && SMALL_WORDS.has(w) ? w : w.charAt(0).toUpperCase() + w.slice(1))) + .join(' '); +} + +/** Longest chat title the Window menu prints before it ellipsizes. */ +const CHAT_TITLE_MAX = 50; + +function shortTitle(title: string): string { + const t = title.trim() || 'Untitled chat'; + return t.length > CHAT_TITLE_MAX ? `${t.slice(0, CHAT_TITLE_MAX - 1)}…` : t; +} + +/** Stem items print a key but never own it — see the note at the top. */ +const display = { registerAccelerator: false } as const; + +export function appMenuTemplate( + mac: boolean, + state: AppMenuState, + actions: AppMenuActions +): MenuItemConstructorOptions[] { + const shortcut = (id: ShortcutId, label?: string): MenuItemConstructorOptions => { + const def = SHORTCUTS.find((d) => d.id === id)!; + return { + label: label ?? titleCase(def.label), + accelerator: chordAccelerator(chordFor(def, mac)), + ...display, + click: () => actions.command({ kind: 'shortcut', id }) + }; + }; + const tab = (id: ManageTabId, label: string): MenuItemConstructorOptions => ({ + label, + click: () => actions.command({ kind: 'manage-tab', tab: id }) + }); + const sep: MenuItemConstructorOptions = { type: 'separator' }; + + const settings = shortcut('open-settings', mac ? 'Settings…' : 'Settings'); + const updates: MenuItemConstructorOptions = { + label: 'Check for Updates…', + enabled: state.canCheckForUpdates, + click: () => actions.checkForUpdates() + }; + const quickChat: MenuItemConstructorOptions = { + label: 'Quick Chat', + ...(state.quickChatAccelerator ? { accelerator: state.quickChatAccelerator, ...display } : {}), + click: () => actions.toggleQuickChat() + }; + + // ⌘1–9 by thread id rather than by digit: a click opens the row the label names. + const chatItems: MenuItemConstructorOptions[] = state.chats.slice(0, 9).map((chat, i) => ({ + label: shortTitle(chat.title), + accelerator: `CmdOrCtrl+${i + 1}`, + ...display, + click: () => actions.command({ kind: 'open-chat', threadId: chat.threadId }) + })); + + const file: MenuItemConstructorOptions = { + label: 'File', + submenu: [ + shortcut('new-conversation'), + shortcut('new-mail'), + quickChat, + sep, + shortcut('attach', 'Attach Files…'), + ...(mac ? [] : [sep, settings, updates, sep, { role: 'quit' } as const]) + ] + }; + + const edit: MenuItemConstructorOptions = { + label: 'Edit', + submenu: [ + { role: 'undo' }, + { role: 'redo' }, + sep, + { role: 'cut' }, + { role: 'copy' }, + { role: 'paste' }, + ...(mac ? [{ role: 'pasteAndMatchStyle' } as const] : []), + { role: 'delete' }, + { role: 'selectAll' }, + sep, + shortcut('focus-chat-search', 'Search Chats'), + ...(mac + ? [ + sep, + { + label: 'Speech', + submenu: [{ role: 'startSpeaking' }, { role: 'stopSpeaking' }] + } as MenuItemConstructorOptions + ] + : []) + ] + }; + + const view: MenuItemConstructorOptions = { + label: 'View', + submenu: [ + tab('chats', 'Chats'), + tab('memory', 'Memory'), + tab('mcp', 'Tools'), + tab('personas', 'Personas'), + tab('folders', 'Sources'), + tab('tasks', 'Scheduled Tasks'), + sep, + shortcut('toggle-inspector', 'Toggle Side Panel'), + shortcut('toggle-format'), + sep, + { role: 'resetZoom' }, + { role: 'zoomIn' }, + { role: 'zoomOut' }, + sep, + { role: 'togglefullscreen' }, + sep, + { + label: 'Developer', + submenu: [{ role: 'reload' }, { role: 'forceReload' }, { role: 'toggleDevTools' }] + } + ] + }; + + const chat: MenuItemConstructorOptions = { + label: 'Chat', + submenu: [ + shortcut('stop'), + sep, + shortcut('cycle-effort'), + shortcut('toggle-speed'), + sep, + shortcut('archive-thread', 'Archive'), + shortcut('snooze-thread', 'Snooze…'), + shortcut('toggle-read'), + sep, + shortcut('delete-thread', 'Delete Chat…') + ] + }; + + const chatsSection = chatItems.length ? [sep, ...chatItems] : []; + // Built by hand rather than with the `windowMenu` role: the role is not + // extensible, and off macOS it is the one that carries Close. + const window: MenuItemConstructorOptions = mac + ? { + role: 'window', + submenu: [{ role: 'minimize' }, { role: 'zoom' }, ...chatsSection, sep, { role: 'front' }] + } + : { label: 'Window', submenu: [{ role: 'minimize' }, ...chatsSection] }; + + if (!mac) return [file, edit, view, chat, window]; + + const stem: MenuItemConstructorOptions = { + label: 'Stem', + submenu: [ + { role: 'about' }, + sep, + settings, + updates, + sep, + { role: 'services' }, + sep, + { role: 'hide' }, + { role: 'hideOthers' }, + { role: 'unhide' }, + sep, + { role: 'quit' } + ] + }; + return [stem, file, edit, view, chat, window]; +} + +let actions: AppMenuActions | null = null; +let state: AppMenuState = { chats: [], quickChatAccelerator: null, canCheckForUpdates: false }; + +/** Install Stem's menu. Call once, after the app is ready. */ +export function installAppMenu(menuActions: AppMenuActions, initial: Partial = {}): void { + actions = menuActions; + state = { ...state, ...initial }; + Menu.setApplicationMenu(Menu.buildFromTemplate(appMenuTemplate(isMac, state, actions))); +} + +/** Rebuild the menu after something it prints changed (the ⌘1–9 rows, the summon key). */ +export function updateAppMenu(patch: Partial): void { + const next = { ...state, ...patch }; + if (JSON.stringify(next) === JSON.stringify(state)) return; + state = next; + if (actions) Menu.setApplicationMenu(Menu.buildFromTemplate(appMenuTemplate(isMac, state, actions))); } diff --git a/src/desktop/browser-host/appimage.ts b/src/desktop/browser-host/appimage.ts new file mode 100644 index 00000000..b781407f --- /dev/null +++ b/src/desktop/browser-host/appimage.ts @@ -0,0 +1,44 @@ +import { closeSync, fstatSync, lstatSync, openSync, readSync, realpathSync } from 'node:fs'; +import { dirname, isAbsolute, normalize } from 'node:path'; + +/** + * May Stem start this AppImage as a browser? The path arrives from a native + * host's word (browser-host/native-host-main.ts reads it from the browser's + * APPIMAGE), and unlike the browsers in shared/browser-native.ts an AppImage + * lives wherever the user put it. So the file itself has to vouch: a regular + * file reached by its real path (no links), executable, with the AppImage + * magic after the ELF header ("AI" and the type byte at offset 8). It and + * every folder above it must be owned by this user or root and writable by + * no one else (a root-owned sticky folder like /tmp is fine), so another + * user cannot swap the file between this check and the launch. + */ +export function launchableAppImage(path: string): boolean { + if (!isAbsolute(path) || normalize(path) !== path) return false; + const uid = process.getuid?.(); + const ownedHere = (owner: number) => owner === 0 || owner === uid; + try { + if (realpathSync(path) !== path) return false; + for (let dir = dirname(path); ; dir = dirname(dir)) { + const st = lstatSync(dir); + const sharedSticky = st.uid === 0 && (st.mode & 0o1000) !== 0; + if (!st.isDirectory() || !ownedHere(st.uid) || ((st.mode & 0o022) !== 0 && !sharedSticky)) return false; + if (dir === '/') break; + } + const fd = openSync(path, 'r'); + try { + const st = fstatSync(fd); + if (!st.isFile() || !ownedHere(st.uid) || (st.mode & 0o100) === 0 || (st.mode & 0o022) !== 0) return false; + const head = Buffer.alloc(11); + if (readSync(fd, head, 0, 11, 0) !== 11) return false; + return ( + head.subarray(0, 4).toString('latin1') === '\x7fELF' && + head.subarray(8, 10).toString('latin1') === 'AI' && + (head[10] === 1 || head[10] === 2) + ); + } finally { + closeSync(fd); + } + } catch { + return false; + } +} diff --git a/src/desktop/browser-host/handshake.ts b/src/desktop/browser-host/handshake.ts new file mode 100644 index 00000000..5affdee3 --- /dev/null +++ b/src/desktop/browser-host/handshake.ts @@ -0,0 +1,33 @@ +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; + +// How the native host and the desktop prove to each other that both hold the +// token Set up wrote to this profile's config, without the token ever crossing +// the socket. Each side sends a fresh nonce and answers the other's with an +// HMAC under the token: +// +// host → desktop host-hello { nonce: h, appPath, appName } +// desktop → host server-hello { nonce: s, proof: mac("stem", h) } +// host → desktop host-proof { proof: mac("host", s) } +// +// The desktop goes first with its proof, so a native host relays nothing, not +// even the extension's hello, to a peer that cannot show the token. That peer +// is the case this exists for: on Windows a named pipe's name can be listed by +// any process, which can take the name once Stem quits and wait for the host's +// reconnect. On a Mac and Linux the socket sits in a 0700 folder, and the same +// handshake costs nothing there. + +export function newNonce(): string { + return randomBytes(16).toString('hex'); +} + +export function handshakeProof(token: string, role: 'stem' | 'host', nonce: string): string { + return createHmac('sha256', token).update(`${role}\0${nonce}`).digest('hex'); +} + +/** Constant-time check of a proof the peer sent. */ +export function proofMatches(token: string, role: 'stem' | 'host', nonce: string, proof: unknown): boolean { + if (typeof proof !== 'string' || !/^[0-9a-f]{64}$/.test(proof)) return false; + return timingSafeEqual(Buffer.from(handshakeProof(token, role, nonce), 'hex'), Buffer.from(proof, 'hex')); +} + +export const isNonce = (v: unknown): v is string => typeof v === 'string' && /^[0-9a-f]{32}$/.test(v); diff --git a/src/desktop/browser-host/index.ts b/src/desktop/browser-host/index.ts index d8ee24e9..fb447bdb 100644 --- a/src/desktop/browser-host/index.ts +++ b/src/desktop/browser-host/index.ts @@ -1,6 +1,6 @@ -import { execFile } from 'node:child_process'; -import { existsSync } from 'node:fs'; -import { randomBytes } from 'node:crypto'; +import { execFile, spawn } from 'node:child_process'; +import { existsSync, readdirSync, readlinkSync } from 'node:fs'; +import { randomBytes, timingSafeEqual } from 'node:crypto'; import { chmod, mkdir, realpath, rm } from 'node:fs/promises'; import { createServer, type Server, type Socket } from 'node:net'; import { dirname, join } from 'node:path'; @@ -10,10 +10,12 @@ import { FrameReader, MAX_TO_EXTENSION_BYTES, type ExtensionAction, + type ExtensionDownload, type ExtensionResult, type FromExtension, type HostHello, - type ToExtension + type ToExtension, + launchableBrowser } from '../../shared/browser-native'; import type { BrowserDownloadReceipt, @@ -33,18 +35,31 @@ import { type BrowserInstallPaths, type InstallSource } from './install'; +import { handshakeProof, isNonce, newNonce, proofMatches } from './handshake'; +import { launchableAppImage } from './appimage'; import { arcSidebarText, readArcSidebar, type ArcSidebarTab } from './arc-tabs'; -import { readBrowserHostSettings, updateBrowserHostSettings, writeBrowserHostEnabled } from './store'; - -// The client half of the `browser` tool: THIS Mac, handing the browser actions +import { + appOf, + readBrowserHostSettings, + updateBrowserHostSettings, + writeBrowserHostEnabled, + type BrowserHostSettings, + type KnownBrowser +} from './store'; + +// The client half of the `browser` tool: THIS Mac, Windows or Linux computer, handing the browser actions // its Stem server sends (BROWSER_REQUEST_FRAME) to the Stem extension in the // user's browser, and answering with what it reports. // // The extension reaches us through its native-messaging host, which the // browser starts and which connects to a unix socket here (0600, in this -// profile's state folder) and opens with the token from the config Set up -// wrote. One connection per browser; several may be up at once (Arc and -// Chrome), and the user picks which one Stem drives. +// profile's state folder; a named pipe on Windows) and proves it holds the +// token from the config Set up wrote (handshake.ts). One connection per +// browser profile — the extension sends an id it keeps in that profile's +// storage — and several may be up at once (Arc's work and personal profiles, +// Chrome beside them). The user picks the default and can name each one; the +// model may name another profile per call, and the one it named stays the +// thread's until it names a different one. // // What this file decides, like computer-host: whether Stem may drive the // browser at all (the switch, read fresh on every request), and the file @@ -90,6 +105,8 @@ export interface BrowserHost { localState(): Promise; setEnabled(enabled: boolean): Promise; choose(id: string): Promise; + rename(id: string, name: string): Promise; + forget(id: string): Promise; setUp(): Promise; openExtensionsPage(browserId?: string): Promise; close(): void; @@ -97,20 +114,168 @@ export interface BrowserHost { interface Conn { socket: Socket; + /** KnownBrowser.id, once the extension said hello; until then the connection is not usable. */ + key: string | null; appPath: string; appName: string; version?: string; pending: Map void>; } -function defaultLaunch(appPath: string): Promise { +/** Longest name a profile may be given. */ +const MAX_PROFILE_NAME = 40; + +/** + * What each known browser is called: the user's name for it, else — when an + * app has several profiles — "Arc 2" by the order they first connected. An + * app's only, unnamed profile has none and goes by the app's name. + */ +export function profileNames(known: readonly KnownBrowser[]): Map { + const names = new Map(); + const seen = new Map(); + for (const b of known) { + const app = appOf(b.id); + const n = (seen.get(app) ?? 0) + 1; + seen.set(app, n); + const siblings = known.filter((o) => appOf(o.id) === app).length; + names.set(b.id, b.label ?? (siblings > 1 ? `${b.name} ${n}` : undefined)); + } + return names; +} + +/** How the model and the popup name a known browser. */ +function nameOf(b: KnownBrowser, names: Map): string { + return names.get(b.id) ?? b.name; +} + +/** The known browser a name from the model means: a profile's name, or an app with one profile. */ +export function findProfile( + known: readonly KnownBrowser[], + wanted: string, + names: Map = profileNames(known) +): KnownBrowser | null { + const w = wanted.trim().toLowerCase(); + if (!w) return null; + const byName = known.find((b) => nameOf(b, names).toLowerCase() === w); + if (byName) return byName; + const byApp = known.filter((b) => b.name.toLowerCase() === w); + return byApp.length === 1 ? byApp[0]! : null; +} + +/** "“Work” (Arc), “Personal” (Arc)" — the names the model may pass. */ +function profileList( + known: readonly KnownBrowser[], + names: Map = profileNames(known) +): string { + return known + .map((b) => { + const name = nameOf(b, names); + return name === b.name ? `“${name}”` : `“${name}” (${b.name})`; + }) + .join(', '); +} + +/** The browser Stem uses on this Mac when nothing else says which. */ +function defaultId(settings: BrowserHostSettings): string | undefined { + const chosen = settings.chosen && settings.known.some((b) => b.id === settings.chosen) ? settings.chosen : undefined; + return chosen ?? settings.known[0]?.id; +} + +/** + * What one request may reach: every known profile, or only those the turn's + * grant allows. Names stay those of the whole list, so "Arc 2" means the same + * profile to every persona. + */ +interface Scope { + /** The settings with `known` cut down to the allowed profiles (the default falls inside them). */ + settings: BrowserHostSettings; + names: Map; + /** The grant limits the profiles. */ + limited: boolean; +} + +export function scopeOf(settings: BrowserHostSettings, allowed: readonly string[] | undefined): Scope { + const names = profileNames(settings.known); + if (!allowed) return { settings, names, limited: false }; + const known = settings.known.filter((b) => allowed.includes(b.id)); + return { settings: { ...settings, known }, names, limited: true }; +} + +/** "mail.google.com, github.com" — the sites a hello says are most open there; anything else is dropped. */ +function asHint(raw: unknown): string | undefined { + if (typeof raw !== 'string') return undefined; + const hosts = raw + .split(',') + .map((h) => h.trim()) + .filter((h) => /^[a-z0-9.-]{1,100}$/i.test(h)) + .slice(0, 3); + return hosts.length ? hosts.join(', ') : undefined; +} + +/** Is a process running this executable? Linux only: it reads /proc. */ +function runningExe(exe: string): boolean { + let pids: string[]; + try { + pids = readdirSync('/proc').filter((n) => /^\d+$/.test(n)); + } catch { + return false; + } + for (const pid of pids) { + try { + if (readlinkSync(`/proc/${pid}/exe`).replace(/ \(deleted\)$/, '') === exe) return true; + } catch { + // quiet: another user's process, or one that just exited. + } + } + return false; +} + +/** Start a program on its own, answering once it started (or failed to). */ +function spawnDetached(command: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { detached: true, stdio: 'ignore' }); + child.once('error', reject); + child.once('spawn', () => { + child.unref(); + resolve(); + }); + }); +} + +/** Is a process running this executable? Windows: one PowerShell question. */ +function runningWindowsExe(exe: string): Promise { + return new Promise((resolve) => { + execFile( + 'powershell.exe', + ['-NoProfile', '-NonInteractive', '-Command', `if (Get-Process | Where-Object { $_.Path -eq $args[0] } | Select-Object -First 1) { 'yes' }`, exe], + { timeout: 10_000, windowsHide: true }, + (error, stdout) => resolve(!error && String(stdout).trim() === 'yes') + ); + }); +} + +async function defaultLaunch(appPath: string, platform: NodeJS.Platform): Promise { + if (platform === 'win32') { + // As on Linux: a second start of a running browser opens a window. + if (await runningWindowsExe(appPath)) return; + return spawnDetached(appPath, []); + } + if (platform === 'linux') { + // Running a browser's executable again while it runs opens a new window; + // the connection is only late, so leave the window count alone. Linux has + // no `open -g`: a browser started here may come to the front. + if (runningExe(appPath)) return Promise.resolve(); + return spawnDetached(appPath, []); + } return new Promise((resolve, reject) => { // -g: do not bring it to the front; the user keeps working where they are. execFile('/usr/bin/open', ['-g', '-a', appPath], (error) => (error ? reject(error) : resolve())); }); } -function defaultOpenWith(appPath: string | null, target: string): Promise { +function defaultOpenWith(appPath: string | null, target: string, platform: NodeJS.Platform): Promise { + if (platform === 'win32') return appPath ? spawnDetached(appPath, [target]) : spawnDetached('explorer.exe', [target]); + if (platform === 'linux') return appPath ? spawnDetached(appPath, [target]) : spawnDetached('xdg-open', [target]); return new Promise((resolve, reject) => { execFile('/usr/bin/open', appPath ? ['-a', appPath, target] : [target], (error) => error ? reject(error) : resolve() @@ -118,6 +283,16 @@ function defaultOpenWith(appPath: string | null, target: string): Promise }); } +const isPipe = (path: string): boolean => path.startsWith('\\\\.\\pipe\\'); + +/** A pre-handshake host-hello carried the token itself; constant-time compare. */ +function legacyTokenMatches(token: string, sent: unknown): boolean { + if (typeof sent !== 'string') return false; + const a = Buffer.from(token); + const b = Buffer.from(sent); + return a.length === b.length && timingSafeEqual(a, b); +} + /** How long one action may take on this side before the extension is presumed stuck. */ export function actionTimeoutMs(action: ExtensionAction): number { switch (action.kind) { @@ -134,15 +309,22 @@ export function actionTimeoutMs(action: ExtensionAction): number { } export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { - const supported = (deps.platform ?? process.platform) === 'darwin'; + const platform = deps.platform ?? process.platform; + const supported = platform === 'darwin' || platform === 'linux' || platform === 'win32'; + // What the model and the user are told this computer is. + const here = platform === 'darwin' ? 'Mac' : 'computer'; const paths = deps.paths ?? browserInstallPaths(); - const launch = deps.launch ?? defaultLaunch; + const launch = deps.launch ?? ((app: string) => defaultLaunch(app, platform)); const launchWaitMs = deps.launchWaitMs ?? LAUNCH_WAIT_MS; - const openWith = deps.openWith ?? defaultOpenWith; + const openWith = deps.openWith ?? ((app: string | null, target: string) => defaultOpenWith(app, target, platform)); + /** A browser Stem knows, or on Linux an AppImage that vouches for itself. */ + const mayLaunch = (app: string): boolean => launchableBrowser(app, platform) || (platform === 'linux' && launchableAppImage(app)); const conns = new Map(); const connectWaiters = new Set<() => void>(); /** Upload folders per thread, removed when the run ends. */ const uploadDirs = new Map>(); + /** The profile each live thread last named (KnownBrowser.id), until its run ends. */ + const threadProfile = new Map(); let server: Server | null = null; let config: BrowserHostConfig | null = null; /** @@ -164,12 +346,22 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { // Known ones first, in the order they first connected, then any connection // whose remembering is still being written. const known = [...settings.known]; - for (const c of conns.values()) if (!known.some((b) => b.id === c.appPath)) known.push({ id: c.appPath, name: c.appName }); + for (const c of conns.values()) if (c.key && !known.some((b) => b.id === c.key)) known.push({ id: c.key, name: c.appName }); + const names = profileNames(known); const browsers: BrowserInfo[] = known.map((b) => { const c = conns.get(b.id); - return { id: b.id, name: b.name, connected: !!c, ...(c?.version ? { version: c.version } : {}) }; + const profile = names.get(b.id); + return { + id: b.id, + name: b.name, + ...(profile ? { profile } : {}), + ...(b.label ? { named: true } : {}), + ...(b.hint ? { hint: b.hint } : {}), + connected: !!c, + ...(c?.version ? { version: c.version } : {}) + }; }); - return { browsers, chosen: settings.chosen ?? browsers[0]?.id ?? null, enabled: settings.enabled }; + return { browsers, chosen: defaultId({ ...settings, known }) ?? null, enabled: settings.enabled }; } async function announce(): Promise { @@ -177,22 +369,86 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { const { browsers, chosen, enabled } = await browsersNow(); await deps .invoke('browserHost:announce', [ - { enabled, platform: 'darwin', browsers, ...(chosen ? { chosen } : {}) } + { + enabled, + platform: platform === 'darwin' || platform === 'win32' ? platform : 'linux', + // The sites open in a profile stay on this computer. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + browsers: browsers.map(({ hint, named, ...b }) => b), + ...(chosen ? { chosen } : {}), + enforcesProfiles: true + } ]) .catch((e) => { log('browser-host', 'could not announce', { error: e instanceof Error ? e.message : String(e) }); }); } + /** Tell every connected extension what Stem calls its profile (a rename can shift the others' "Arc 2"). */ + async function sendNames(): Promise { + const known = (await readBrowserHostSettings()).known; + const names = profileNames(known); + for (const b of known) { + const c = conns.get(b.id); + if (c) send(c, { type: 'profile', name: nameOf(b, names) }); + } + } + + /** + * Remember the profile behind a hello. An entry from before profiles were + * told apart (the bare app path) is taken over by the first profile of that + * app to connect, keeping its place as the default. + */ + async function remember(conn: Conn, key: string, hint: string | undefined): Promise { + await updateBrowserHostSettings((cur) => { + const at = cur.known.findIndex((b) => b.id === key); + if (at >= 0) { + const known = [...cur.known]; + known[at] = { ...known[at]!, name: conn.appName, ...(hint ? { hint } : {}) }; + return { ...cur, known }; + } + const entry: KnownBrowser = { id: key, name: conn.appName, ...(hint ? { hint } : {}) }; + const legacy = key !== conn.appPath ? cur.known.findIndex((b) => b.id === conn.appPath) : -1; + if (legacy >= 0 && !conns.has(conn.appPath)) { + const known = [...cur.known]; + known[legacy] = { ...entry, ...(known[legacy]!.label ? { label: known[legacy]!.label } : {}) }; + return { ...cur, known, ...(cur.chosen === conn.appPath ? { chosen: key } : {}) }; + } + return { ...cur, known: [...cur.known, entry] }; + }); + } + + async function onHello(conn: Conn, message: Extract): Promise { + conn.version = typeof message.extensionVersion === 'string' ? message.extensionVersion : undefined; + const profile = + typeof message.profileId === 'string' && /^[A-Za-z0-9-]{8,64}$/.test(message.profileId) ? message.profileId : null; + // An extension from before 0.7.0 sends no profile id: one entry per app, as then. + const key = profile ? `${conn.appPath}#${profile}` : conn.appPath; + log('browser-host', 'the extension said hello', { browser: conn.appName, version: conn.version, profile: !!profile }); + if (conn.key !== key) { + if (conn.key && conns.get(conn.key) === conn) conns.delete(conn.key); + // A browser that restarted connects again before the old socket is + // noticed gone: the new connection wins. + const old = conns.get(key); + if (old && old !== conn) old.socket.destroy(); + conn.key = key; + conns.set(key, conn); + } + if (reloadOwed.delete(key) || reloadOwed.delete(conn.appPath)) { + send(conn, { type: 'reload' }); + return; + } + await remember(conn, key, asHint(message.hint)).catch((e) => + log('browser-host', 'could not remember a browser', { error: String(e) }) + ); + for (const wake of [...connectWaiters]) wake(); + await sendNames(); + await announce(); + } + function onFrame(conn: Conn, message: FromExtension): void { if (message.type === 'hello') { - conn.version = typeof message.extensionVersion === 'string' ? message.extensionVersion : undefined; - log('browser-host', 'the extension said hello', { browser: conn.appName, version: conn.version }); - if (reloadOwed.delete(conn.appPath)) { - send(conn, { type: 'reload' }); - return; - } - void announce(); + void onHello(conn, message); return; } if (message.type === 'result') { @@ -213,6 +469,8 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { function accept(socket: Socket): void { const reader = new FrameReader(); let conn: Conn | null = null; + /** Between the host's hello and its proof: what it claimed, and the nonce it must answer. */ + let pending: { appPath: string; appName: string; nonce: string } | null = null; const hello = setTimeout(() => socket.destroy(), 5000); hello.unref?.(); socket.on('data', (chunk: Buffer) => { @@ -230,31 +488,50 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { } catch { continue; } - if (!conn) { - const h = message as Partial; - if (h?.type !== 'host-hello' || !config || h.token !== config.token || typeof h.appPath !== 'string') { + if (!conn && !pending) { + const h = message as Partial & { token?: unknown }; + if (h?.type === 'host-hello' && config && !('nonce' in h) && legacyTokenMatches(config.token, h.token)) { + // A native host from before the handshake (Stem ≤ 0.6), still + // running because the browser kept its port open across a Stem + // update: it would retry every two seconds forever and the + // profile would never connect (2026-10-19). It relays whatever + // comes back unchecked, so tell the extension to reload — that + // closes the port, the old host exits, and the browser starts the + // one on disk. + log('browser-host', 'an old native host connected; reloading the extension so the browser starts the new one'); + clearTimeout(hello); + socket.end(encodeFrame({ type: 'reload' } satisfies ToExtension)); + return; + } + if (h?.type !== 'host-hello' || !config || !isNonce(h.nonce) || typeof h.appPath !== 'string') { + log('browser-host', 'refused a native host that did not start the handshake'); + socket.destroy(); + return; + } + pending = { + appPath: h.appPath, + appName: typeof h.appName === 'string' && h.appName ? h.appName : 'Browser', + nonce: newNonce() + }; + socket.write(encodeFrame({ type: 'server-hello', nonce: pending.nonce, proof: handshakeProof(config.token, 'stem', h.nonce) })); + continue; + } + if (!conn && pending) { + const p = message as { type?: unknown; proof?: unknown }; + if (p?.type !== 'host-proof' || !config || !proofMatches(config.token, 'host', pending.nonce, p.proof)) { log('browser-host', 'refused a native host without the right token'); socket.destroy(); return; } clearTimeout(hello); - const appPath = h.appPath; - const appName = typeof h.appName === 'string' && h.appName ? h.appName : 'Browser'; - // A browser that restarted connects again before the old socket is - // noticed gone: the new connection wins. - conns.get(appPath)?.socket.destroy(); - conn = { socket, appPath, appName, pending: new Map() }; - conns.set(appPath, conn); - void updateBrowserHostSettings((cur) => - cur.known.some((b) => b.id === appPath) - ? cur - : { ...cur, known: [...cur.known, { id: appPath, name: appName }] } - ).then(() => { - for (const wake of [...connectWaiters]) wake(); - }); + const { appPath, appName } = pending; + pending = null; + // Which profile this is arrives with the extension's own hello, + // right behind (the native host replays it on every connection). + conn = { socket, key: null, appPath, appName, pending: new Map() }; continue; } - onFrame(conn, message as FromExtension); + if (conn) onFrame(conn, message as FromExtension); } }); socket.on('error', () => undefined); @@ -270,8 +547,8 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { }); } conn.pending.clear(); - if (conns.get(conn.appPath) === conn) { - conns.delete(conn.appPath); + if (conn.key && conns.get(conn.key) === conn) { + conns.delete(conn.key); void announce(); } }); @@ -282,36 +559,59 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { config = await ensureHostConfig(paths); await mkdir(paths.stateDir, { recursive: true, mode: 0o700 }); // A socket file left by a previous run (or a crash) would make listen fail. - await rm(config.socketPath, { force: true }); + // A Windows pipe goes away with its last handle. + if (!isPipe(config.socketPath)) await rm(config.socketPath, { force: true }); const s = createServer(accept); await new Promise((resolve, reject) => { s.once('error', reject); s.listen(config!.socketPath, () => resolve()); }); - await chmod(config.socketPath, 0o600).catch(() => undefined); + if (!isPipe(config.socketPath)) await chmod(config.socketPath, 0o600).catch(() => undefined); s.on('error', (e) => log('browser-host', 'socket error', { error: String(e) })); server = s; } - /** The connection for the chosen browser, launching it when it is not running. */ - async function connectionFor(): Promise<{ ok: true; conn: Conn } | { ok: false; error: string }> { - const settings = await readBrowserHostSettings(); - const chosen = settings.chosen ?? settings.known[0]?.id ?? [...conns.keys()][0]; + /** The connection for a known browser (the default when none is named), launching it when it is not running. */ + async function connectionFor( + settings: BrowserHostSettings, + id?: string, + scope?: Scope + ): Promise<{ ok: true; conn: Conn } | { ok: false; error: string }> { + const chosen = id ?? defaultId(scope?.settings ?? settings) ?? (scope?.limited ? undefined : [...conns.keys()][0]); + if (!chosen && scope?.limited) { + return { + ok: false, + error: + 'This conversation may use only some of the browser profiles on this Mac, and none of those is set up ' + + 'there any more. Tell the user to check which profiles it may use (the persona’s browser pin, or ' + + 'Settings → Features → Browser control for chats).' + }; + } if (!chosen) { return { ok: false, error: - 'No browser with the Stem extension has connected on this Mac yet. Tell the user to set it up in ' + - 'Settings → Features → Browser control → Set up, on that Mac.' + `No browser with the Stem extension has connected on this ${here} yet. Tell the user to set it up in ` + + `Settings → Features → Browser control → Set up, on that ${here}.` }; } const live = conns.get(chosen); if (live) return { ok: true, conn: live }; - const name = settings.known.find((b) => b.id === chosen)?.name ?? chosen; + const entry = settings.known.find((b) => b.id === chosen); + const app = appOf(chosen); + const profiles = settings.known.filter((b) => appOf(b.id) === app).length; + const name = entry ? nameOf(entry, profileNames(settings.known)) : chosen; + if (!mayLaunch(app)) { + // The id came from a native host's word; Stem starts only browsers it knows. + return { + ok: false, + error: `${name} is not running, and Stem only starts browsers it knows on this ${here}. Tell the user to open it.` + }; + } try { - await launch(chosen); + await launch(app); } catch (e) { - return { ok: false, error: `${name} could not be started on this Mac: ${e instanceof Error ? e.message : String(e)}` }; + return { ok: false, error: `${name} could not be started on this ${here}: ${e instanceof Error ? e.message : String(e)}` }; } const conn = await new Promise((resolve) => { const wake = (): void => { @@ -335,14 +635,112 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { log('browser-host', 'started the browser for a run', { browser: name }); return { ok: true, conn }; } + if (profiles > 1) { + return { + ok: false, + error: + `The browser profile “${name}” is not open, and starting ${entry?.name ?? 'the browser'} did not bring it up ` + + `within ${Math.round(launchWaitMs / 1000)}s. Ask the user to open a window of that profile, or use another profile.` + }; + } return { ok: false, error: `${name} was started, but the Stem extension did not connect within ${Math.round(launchWaitMs / 1000)}s. It may be ` + - 'disabled or removed there; tell the user to check Settings → Features → Browser control on that Mac.' + `disabled or removed there; tell the user to check Settings → Features → Browser control on that ${here}.` }; } + /** One action on one connection. */ + function ask(conn: Conn, request: DeviceBrowserRequest, action: ExtensionAction): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => { + conn.pending.delete(request.requestId); + resolve({ + ok: false, + error: + `${conn.appName} did not finish the action in time. It may or may not have happened: check the page ` + + '(snapshot) before repeating it.' + }); + }, actionTimeoutMs(action)); + conn.pending.set(request.requestId, (r) => { + clearTimeout(timer); + resolve(r); + }); + const sent = send(conn, { type: 'request', id: request.requestId, threadId: request.threadId, action }); + if (!sent) { + clearTimeout(timer); + conn.pending.delete(request.requestId); + resolve({ ok: false, error: 'The action is too large to hand to the browser (over 1 MB); shorten it.' }); + } + }); + } + + /** Arc's sidebar tabs the extension could not list, as lines to add to a `tabs` answer. */ + async function arcExtra(loadedUrls: string[]): Promise { + const sidebar = await (deps.readArcSidebar ?? readArcSidebar)(); + return 'error' in sidebar ? sidebar.error : arcSidebarText(sidebar.tabs, loadedUrls); + } + + /** + * `tabs` with no profile named, on a Mac with several: every open profile's + * tabs, under its name, so the model can find a page wherever it is. The + * thread's own profile is listed (and started) even when it is closed. + */ + async function tabsEverywhere( + request: DeviceBrowserRequest, + full: BrowserHostSettings, + scope: Scope, + current: string | undefined + ): Promise { + const { settings, names } = scope; + const targets: { entry: KnownBrowser; conn: Conn | null; error?: string }[] = []; + for (const entry of settings.known) { + const live = conns.get(entry.id) ?? null; + if (live) targets.push({ entry, conn: live }); + else if (entry.id === current) { + const opened = await connectionFor(full, entry.id); + targets.push(opened.ok ? { entry, conn: opened.conn } : { entry, conn: null, error: opened.error }); + } + } + const answers = await Promise.all( + targets.map(async (t) => (t.conn ? ask(t.conn, request, { kind: 'tabs' }) : ({ ok: false, error: t.error ?? '' } as const))) + ); + const sections: string[] = []; + const arcUrls: string[] = []; + const downloads: ExtensionDownload[] = []; + let arc = false; + targets.forEach((t, i) => { + const r = answers[i]!; + const marks = [t.entry.id === current ? 'your current profile' : '', t.entry.id === defaultId(full) ? 'default' : ''] + .filter(Boolean) + .join(', '); + const name = nameOf(t.entry, names); + const head = `## Profile “${name}”${name === t.entry.name ? '' : ` (${t.entry.name})`}${marks ? ` — ${marks}` : ''}`; + sections.push(`${head}\n${r.ok ? (r.text ?? '') : `Its tabs could not be listed: ${r.error}`}`); + if (r.ok && r.downloads) downloads.push(...r.downloads); + if (t.conn && /\/Arc\.app$/.test(t.conn.appPath)) { + arc = true; + if (r.ok) arcUrls.push(...(r.urls ?? [])); + } + }); + const closed = settings.known.filter((b) => !targets.some((t) => t.entry.id === b.id)); + const lines = [ + `${scope.limited ? 'This conversation may use' : "This Mac's browser has"} ${settings.known.length} profiles ` + + `with Stem: ${profileList(settings.known, names)}. A tab id ` + + 'belongs to its profile: to work in a tab of another profile than your current one, pass that `profile` too ' + + '(it then stays your profile for later calls).', + ...sections, + ...(closed.length ? [`Not open right now: ${profileList(closed, names)}.`] : []) + ]; + if (arc) { + const extra = await arcExtra(arcUrls); + if (extra) lines.push(extra); + } + if (!answers.some((r) => r.ok)) return { ok: false, error: lines.join('\n\n') }; + return { ok: true, text: lines.join('\n\n'), ...(downloads.length ? { downloads } : {}) }; + } + /** The device's action, as the extension takes it: an upload's files fetched to this Mac first. */ async function localAction(request: DeviceBrowserRequest): Promise { const action = request.action; @@ -378,14 +776,14 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { const real = spool ? await realpath(d.path).catch(() => null) : null; const folder = real ? dirname(real) : null; if (!spool || !real || !folder || dirname(folder) !== spool) { - notes.push(`Download “${d.name}” finished on the Mac but could not be copied for Stem; ask the user for the file.`); + notes.push(`Download “${d.name}” finished on the ${here} but could not be copied for Stem; ask the user for the file.`); continue; } try { const handle = await deps.uploadFile(real); receipts.push({ handle, name: d.name, size: d.size, ...(d.mime ? { mime: d.mime } : {}) }); } catch (e) { - notes.push(`Download “${d.name}” finished on the Mac but could not be sent to Stem: ${e instanceof Error ? e.message : String(e)}`); + notes.push(`Download “${d.name}” finished on the ${here} but could not be sent to Stem: ${e instanceof Error ? e.message : String(e)}`); } // The spool copy has done its job either way. void rm(folder, { recursive: true, force: true }); @@ -395,48 +793,62 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { } async function execute(request: DeviceBrowserRequest): Promise { - if (!supported) return { ok: false, error: 'Browser control is only available on macOS for now.' }; + if (!supported) return { ok: false, error: 'Browser control is only available on macOS, Windows and Linux.' }; if (!(await readBrowserHostSettings()).enabled) { return { ok: false, error: - 'This Mac does not let Stem drive its browser. The switch is in Settings → Features → Browser control, ' + + `This ${here} does not let Stem drive its browser. The switch is in Settings → Features → Browser control, ` + 'on that computer.' }; } - const target = await connectionFor(); + const full = await readBrowserHostSettings(); + // The persona's pin or the chat setting may limit the profiles: outside + // them a profile is not listed, not found by name and never a fallback. + const allowed = Array.isArray(request.allowed) + ? request.allowed.filter((a): a is string => typeof a === 'string') + : undefined; + const scope = scopeOf(full, allowed); + const settings = scope.settings; + // Which profile: the one named in this call, else the one this thread + // named last, else the default. + let id: string | undefined; + const wanted = typeof request.profile === 'string' ? request.profile.trim() : ''; + if (wanted) { + const hit = findProfile(settings.known, wanted, scope.names); + if (!hit) { + const what = scope.limited ? 'This conversation may use no browser profile called' : 'This Mac has no browser profile called'; + const list = scope.limited ? 'The ones it may use' : 'Its profiles'; + return { + ok: false, + error: settings.known.length + ? `${what} “${wanted.slice(0, 60)}”. ${list}: ${profileList(settings.known, scope.names)}.` + : scope.limited + ? `This conversation may use no browser profile called “${wanted.slice(0, 60)}”.` + : `No browser has connected the Stem extension on this Mac yet, so there is no profile “${wanted.slice(0, 60)}”.` + }; + } + id = hit.id; + threadProfile.set(request.threadId, id); + } else { + const last = threadProfile.get(request.threadId); + id = last && settings.known.some((b) => b.id === last) ? last : undefined; + } + if (request.action.kind === 'tabs' && !wanted && settings.known.length > 1) { + return remoteResult(await tabsEverywhere(request, full, scope, id ?? defaultId(settings))); + } + const target = await connectionFor(full, id ?? defaultId(settings), scope); if (!target.ok) return { ok: false, error: target.error }; const conn = target.conn; let action: ExtensionAction; try { action = await localAction(request); } catch (e) { - return { ok: false, error: `The files could not be brought to the Mac for the upload: ${e instanceof Error ? e.message : String(e)}` }; + return { ok: false, error: `The files could not be brought to the ${here} for the upload: ${e instanceof Error ? e.message : String(e)}` }; } - const result = await new Promise((resolve) => { - const timer = setTimeout(() => { - conn.pending.delete(request.requestId); - resolve({ - ok: false, - error: - `${conn.appName} did not finish the action in time. It may or may not have happened: check the page ` + - '(snapshot) before repeating it.' - }); - }, actionTimeoutMs(action)); - conn.pending.set(request.requestId, (r) => { - clearTimeout(timer); - resolve(r); - }); - const sent = send(conn, { type: 'request', id: request.requestId, threadId: request.threadId, action }); - if (!sent) { - clearTimeout(timer); - conn.pending.delete(request.requestId); - resolve({ ok: false, error: 'The action is too large to hand to the browser (over 1 MB); shorten it.' }); - } - }); + const result = await ask(conn, request, action); if (action.kind === 'tabs' && result.ok && /\/Arc\.app$/.test(conn.appPath)) { - const sidebar = await (deps.readArcSidebar ?? readArcSidebar)(); - const extra = 'error' in sidebar ? sidebar.error : arcSidebarText(sidebar.tabs, result.urls ?? []); + const extra = await arcExtra(result.urls ?? []); if (extra) result.text = [result.text, extra].filter(Boolean).join('\n\n'); } return remoteResult(result); @@ -450,7 +862,7 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { async function refreshInstall(): Promise { try { - const refreshed = await refreshBrowserControl(deps.installSource(), paths, deps.home); + const refreshed = await refreshBrowserControl(deps.installSource(), paths, deps.home, platform); if (refreshed?.extensionChanged) { await oweReload(); } @@ -467,7 +879,7 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { browsers, chosen, extensionPath: existsSync(paths.extensionDir) ? paths.extensionDir : null, - setupBrowsers: supported ? setupBrowsers(deps.home) : [] + setupBrowsers: supported ? setupBrowsers(deps.home, platform) : [] }; } @@ -502,6 +914,7 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { }, onEnd({ threadId }) { + threadProfile.delete(threadId); for (const c of conns.values()) send(c, { type: 'end', threadId }); for (const dir of uploadDirs.get(threadId) ?? []) void rm(dir, { recursive: true, force: true }); uploadDirs.delete(threadId); @@ -522,10 +935,47 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { return state(); }, + async rename(id, name) { + const label = name.trim().replace(/\s+/g, ' ').slice(0, MAX_PROFILE_NAME); + await updateBrowserHostSettings((cur) => { + if (!cur.known.some((b) => b.id === id)) return cur; + if (label && cur.known.some((b) => b.id !== id && b.label?.toLowerCase() === label.toLowerCase())) { + throw new Error(`Another browser profile is already called “${label}”.`); + } + return { + ...cur, + known: cur.known.map((b) => { + if (b.id !== id) return b; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { label: _old, ...rest } = b; + return label ? { ...rest, label } : rest; + }) + }; + }); + await sendNames(); + await announce(); + return state(); + }, + + async forget(id) { + if (conns.has(id)) return state(); + await updateBrowserHostSettings((cur) => { + const known = cur.known.filter((b) => b.id !== id); + if (cur.chosen !== id) return { ...cur, known }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { chosen: _gone, ...rest } = cur; + return { ...rest, known }; + }); + reloadOwed.delete(id); + await sendNames(); + await announce(); + return state(); + }, + async setUp() { if (!supported) return state(); await listen().catch((e) => log('browser-host', 'could not listen for the extension', { error: String(e) })); - const result = await installBrowserControl(deps.installSource(), paths, deps.home); + const result = await installBrowserControl(deps.installSource(), paths, deps.home, platform); log('browser-host', 'set up browser control', { registered: result.registered }); if (result.extensionChanged) { await oweReload(); @@ -535,20 +985,24 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { async openExtensionsPage(browserId) { const settings = await readBrowserHostSettings(); - const app = browserId ?? settings.chosen ?? settings.known[0]?.id ?? setupBrowsers(deps.home)[0]?.app ?? null; + const known = browserId ?? defaultId(settings); + const app = (known ? appOf(known) : null) ?? setupBrowsers(deps.home, platform)[0]?.app ?? null; // The path goes on the clipboard first, while Stem still has focus: Load - // unpacked's picker takes it with ⌘⇧G, which beats hunting for a folder - // inside Application Support. + // unpacked's picker takes it with ⌘⇧G (Ctrl+L in GTK's), which beats + // hunting for a folder inside Application Support or ~/.config. try { deps.copyText?.(paths.extensionDir); } catch (e) { log('browser-host', 'could not copy the extension folder', { error: String(e) }); } // Browsers refuse chrome:// URLs from outside, so the reliable half is - // the folder: Finder shows it, ready for Load unpacked. The browser still + // the folder: Finder (or the file manager) shows it, ready for Load unpacked. The browser still // comes forward, so typing its extensions page is all that is left. await openWith(null, paths.extensionDir).catch(() => undefined); - if (app) await openWith(app, extensionsPageUrl(app)).catch(() => undefined); + // The renderer names the app; on Linux that is a program to run, so only a known browser. + if (app && mayLaunch(app)) { + await openWith(app, extensionsPageUrl(app)).catch(() => undefined); + } }, close() { @@ -556,7 +1010,7 @@ export function createBrowserHost(deps: BrowserHostDeps): BrowserHost { conns.clear(); server?.close(); server = null; - if (config) void rm(config.socketPath, { force: true }); + if (config && !isPipe(config.socketPath)) void rm(config.socketPath, { force: true }); } }; } diff --git a/src/desktop/browser-host/install.ts b/src/desktop/browser-host/install.ts index 92e0e903..ea0d5353 100644 --- a/src/desktop/browser-host/install.ts +++ b/src/desktop/browser-host/install.ts @@ -3,7 +3,7 @@ import { createHash, randomBytes } from 'node:crypto'; import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; import { chmod, cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { homedir, tmpdir } from 'node:os'; -import { dirname, join, relative } from 'node:path'; +import { dirname, join, relative, sep } from 'node:path'; import { host } from '../../server/host'; import { BROWSER_EXTENSION_ID, BROWSER_NATIVE_HOST } from '../../shared/browser-native'; import type { SetupBrowser } from '../../shared/types'; @@ -21,7 +21,9 @@ import type { SetupBrowser } from '../../shared/types'; // /browser/uploads/ files fetched from the server for one upload // // The manifests are per BROWSER, not per profile — one com.stem.browser per -// browser — so the last profile to run Set up owns them. Startup refreshes them +// browser — so the last profile to run Set up owns them. On Windows a browser +// finds its manifest through the registry instead of a folder: one manifest +// file in the state folder, and a key per browser under HKCU pointing at it. Startup refreshes them // only when they already point at this profile's wrapper, so an isolated dev // instance (STEM_PROFILE) never takes over the real install by being launched. @@ -30,6 +32,8 @@ export interface BrowserInstallPaths { extensionDir: string; configPath: string; wrapperPath: string; + /** Windows: the one manifest every browser's registry key points at. */ + manifestPath: string; spoolDir: string; uploadsDir: string; } @@ -40,20 +44,29 @@ export interface BrowserHostConfig { spoolDir: string; } -export function browserInstallPaths(stateRoot = host().stateRoot()): BrowserInstallPaths { +export function browserInstallPaths(stateRoot = host().stateRoot(), platform: NodeJS.Platform = process.platform): BrowserInstallPaths { const stateDir = join(stateRoot, 'browser'); return { stateDir, extensionDir: join(stateRoot, 'browser-extension'), configPath: join(stateDir, 'config.json'), - wrapperPath: join(stateDir, 'stem-browser-host'), + // A Windows browser starts a manifest's `path` through cmd.exe, so it is a batch file there. + wrapperPath: join(stateDir, platform === 'win32' ? 'stem-browser-host.cmd' : 'stem-browser-host'), + manifestPath: join(stateDir, `${BROWSER_NATIVE_HOST}.json`), spoolDir: join(stateDir, 'spool'), uploadsDir: join(stateDir, 'uploads') }; } -/** Unix sockets cap their path near 104 bytes; a long profile path falls back to a short one in tmp. */ -export function socketPathFor(stateDir: string): string { +/** + * Unix sockets cap their path near 104 bytes; a long profile path falls back + * to a short one in tmp. Windows has no unix sockets in Node: a named pipe, + * whose name is random and kept in the config. Pipe names are one namespace for + * the whole machine, so a name anyone could work out could be taken by another + * process before Stem starts, and the native host would hand it the token. + */ +export function socketPathFor(stateDir: string, platform: NodeJS.Platform = process.platform): string { + if (platform === 'win32') return `\\\\.\\pipe\\stem-browser-${randomBytes(16).toString('hex')}`; const preferred = join(stateDir, 'host.sock'); if (Buffer.byteLength(preferred) < 100) return preferred; const tag = createHash('sha256').update(stateDir).digest('hex').slice(0, 16); @@ -85,78 +98,195 @@ export async function ensureHostConfig(paths = browserInstallPaths()): Promise.app` in an Applications folder, or a command on PATH. */ + launchers: string[]; + /** How the system names it as the default browser: a bundle id, or a .desktop file. */ + defaultIds: string[]; /** Present when the browser has ever run for this user. */ dataDir: string; hostsDir: string; + /** Windows: the HKCU key the browser reads native-messaging hosts from. */ + registryKey?: string; } -export function browserDirs(home = homedir()): BrowserDir[] { +const dir = (name: string, app: string, launchers: string[], defaultIds: string[], dataDir: string): BrowserDir => ({ + name, + app, + launchers, + defaultIds, + dataDir, + hostsDir: join(dataDir, 'NativeMessagingHosts') +}); + +function macBrowserDirs(home: string): BrowserDir[] { const support = join(home, 'Library', 'Application Support'); - const chrome = join(support, 'Google', 'Chrome'); - const dirs: BrowserDir[] = [ - { name: 'Google Chrome', app: 'Google Chrome', bundleId: 'com.google.Chrome', dataDir: chrome, hostsDir: join(chrome, 'NativeMessagingHosts') }, - { - name: 'Arc', - app: 'Arc', - bundleId: 'company.thebrowser.Browser', - dataDir: join(support, 'Arc', 'User Data'), - hostsDir: join(support, 'Arc', 'User Data', 'NativeMessagingHosts') - }, - { - name: 'Dia', - app: 'Dia', - bundleId: 'company.thebrowser.dia', - dataDir: join(support, 'Dia', 'User Data'), - hostsDir: join(support, 'Dia', 'User Data', 'NativeMessagingHosts') - }, - { - name: 'Brave', - app: 'Brave Browser', - bundleId: 'com.brave.Browser', - dataDir: join(support, 'BraveSoftware', 'Brave-Browser'), - hostsDir: join(support, 'BraveSoftware', 'Brave-Browser', 'NativeMessagingHosts') - }, - { - name: 'Chromium', - app: 'Chromium', - bundleId: 'org.chromium.Chromium', - dataDir: join(support, 'Chromium'), - hostsDir: join(support, 'Chromium', 'NativeMessagingHosts') - }, - { - name: 'Microsoft Edge', - app: 'Microsoft Edge', - bundleId: 'com.microsoft.edgemac', - dataDir: join(support, 'Microsoft Edge'), - hostsDir: join(support, 'Microsoft Edge', 'NativeMessagingHosts') - }, - { - name: 'Google Chrome for Testing', - app: 'Google Chrome for Testing', - bundleId: 'com.google.chrome.for.testing', - dataDir: join(support, 'Google', 'Chrome for Testing'), - hostsDir: join(support, 'Google', 'Chrome for Testing', 'NativeMessagingHosts') - } + const mac = (name: string, app: string, bundleId: string, ...data: string[]) => + dir(name, app, [`${app}.app`], [bundleId], join(support, ...data)); + const dirs = [ + mac('Google Chrome', 'Google Chrome', 'com.google.Chrome', 'Google', 'Chrome'), + mac('Arc', 'Arc', 'company.thebrowser.Browser', 'Arc', 'User Data'), + mac('Dia', 'Dia', 'company.thebrowser.dia', 'Dia', 'User Data'), + mac('Brave', 'Brave Browser', 'com.brave.Browser', 'BraveSoftware', 'Brave-Browser'), + mac('Chromium', 'Chromium', 'org.chromium.Chromium', 'Chromium'), + mac('Microsoft Edge', 'Microsoft Edge', 'com.microsoft.edgemac', 'Microsoft Edge'), + mac('Google Chrome for Testing', 'Google Chrome for Testing', 'com.google.chrome.for.testing', 'Google', 'Chrome for Testing') ]; // Arc has been seen reading Chrome's folder rather than its own, so with Arc // present Chrome's folder gets the manifest too, installed or not. + const chrome = dirs[0]!.dataDir; const arcPresent = existsSync(dirs[1]!.dataDir); return dirs.filter((d) => existsSync(d.dataDir) || (arcPresent && d.dataDir === chrome)); } -/** The address of a browser's extensions page, by app name or bundle path. */ +/** + * Linux browsers keep their per-user manifests under the XDG config folder. + * Snap and Flatpak builds are not here: their sandbox reads another folder and + * will not start a host outside it. + */ +function linuxBrowserDirs(home: string): BrowserDir[] { + const config = home === homedir() && process.env.XDG_CONFIG_HOME ? process.env.XDG_CONFIG_HOME : join(home, '.config'); + const dirs = [ + // Every launcher here must be in LINUX_BROWSER_COMMANDS, or Stem will not run it. + dir('Google Chrome', 'google-chrome', ['google-chrome', 'google-chrome-stable'], ['google-chrome.desktop'], join(config, 'google-chrome')), + dir('Brave', 'brave-browser', ['brave-browser', 'brave'], ['brave-browser.desktop', 'com.brave.Browser.desktop'], join(config, 'BraveSoftware', 'Brave-Browser')), + dir('Chromium', 'chromium', ['chromium', 'chromium-browser'], ['chromium.desktop', 'chromium-browser.desktop'], join(config, 'chromium')), + dir('Microsoft Edge', 'microsoft-edge', ['microsoft-edge', 'microsoft-edge-stable'], ['microsoft-edge.desktop'], join(config, 'microsoft-edge')), + ]; + const known = dirs.filter((d) => existsSync(d.dataDir)); + const taken = new Set(dirs.map((d) => d.dataDir)); + // Any other Chromium browser — an AppImage (Helium, Thorium), a tarball, a + // beta channel, Vivaldi, Opera — reads /NativeMessagingHosts + // too. It has no command Stem knows; Set up only writes its manifest. + const others = chromiumDataDirs(config) + .filter((d) => !taken.has(d)) + .map((d) => dir(otherBrowserName(relative(config, d)), '', [], [], d)); + return [...known, ...others]; +} + +/** Folder names that do not say the browser's name. */ +const OTHER_BROWSER_NAMES: Record = { + 'net.imput.helium': 'Helium', + 'google-chrome-beta': 'Google Chrome Beta', + 'google-chrome-unstable': 'Google Chrome Dev', + 'microsoft-edge-beta': 'Microsoft Edge Beta', + 'microsoft-edge-dev': 'Microsoft Edge Dev' +}; + +/** "BraveSoftware/Brave-Browser-Beta" → "Brave Browser Beta", "thorium" → "Thorium". */ +function otherBrowserName(rel: string): string { + if (OTHER_BROWSER_NAMES[rel]) return OTHER_BROWSER_NAMES[rel]!; + const last = rel.split(sep).pop() ?? rel; + return last + .split(/[-_ .]+/) + .filter(Boolean) + .map((w) => w[0]!.toUpperCase() + w.slice(1)) + .join(' '); +} + +/** + * Chromium user-data folders one or two levels under the config folder: a + * `Local State` file and something only a browser writes beside it (a fresh + * Helium has `Variations` and `Crash Reports` before its `Default` profile). + * Electron apps keep a Local State too, but their folder is itself the + * profile, `Preferences` and all, and they crash into `Crashpad`. + */ +const BROWSER_MARKS = ['Default', 'Crash Reports', 'BrowserMetrics', 'Variations']; + +function chromiumDataDirs(config: string): string[] { + const isData = (d: string) => + existsSync(join(d, 'Local State')) && + !existsSync(join(d, 'Preferences')) && + BROWSER_MARKS.some((m) => existsSync(join(d, m))); + const subdirs = (d: string): string[] => { + try { + return readdirSync(d, { withFileTypes: true }) + .filter((e) => e.isDirectory() && !e.name.startsWith('.')) + .map((e) => join(d, e.name)); + } catch { + return []; + } + }; + const found: string[] = []; + for (const d of subdirs(config)) { + if (isData(d)) found.push(d); + else found.push(...subdirs(d).filter(isData)); + } + return found; +} + +/** + * Windows browsers keep their data under AppData\\Local and read host + * manifests from the registry. `launchers` are the executables an installer + * puts down, machine-wide or per user; the first that exists is what Stem runs. + */ +function windowsBrowserDirs(home: string): BrowserDir[] { + const local = home === homedir() && process.env.LOCALAPPDATA ? process.env.LOCALAPPDATA : join(home, 'AppData', 'Local'); + const programs = [process.env.ProgramFiles ?? 'C:\\Program Files', process.env['ProgramFiles(x86)'] ?? 'C:\\Program Files (x86)']; + const exes = (...tail: string[]) => [...programs, local].map((root) => join(root, ...tail)); + const win = (name: string, defaultId: string, key: string, launchers: string[], ...data: string[]): BrowserDir => ({ + name, + app: launchers[0]!, + launchers, + defaultIds: [defaultId], + dataDir: join(local, ...data), + hostsDir: join(local, ...data), + registryKey: `HKCU\\Software\\${key}\\NativeMessagingHosts` + }); + const dirs = [ + win('Google Chrome', 'ChromeHTML', 'Google\\Chrome', exes('Google', 'Chrome', 'Application', 'chrome.exe'), 'Google', 'Chrome', 'User Data'), + win('Microsoft Edge', 'MSEdgeHTM', 'Microsoft\\Edge', exes('Microsoft', 'Edge', 'Application', 'msedge.exe'), 'Microsoft', 'Edge', 'User Data'), + win('Brave', 'BraveHTML', 'BraveSoftware\\Brave-Browser', exes('BraveSoftware', 'Brave-Browser', 'Application', 'brave.exe'), 'BraveSoftware', 'Brave-Browser', 'User Data'), + win('Chromium', 'ChromiumHTM', 'Chromium', [join(local, 'Chromium', 'Application', 'chrome.exe')], 'Chromium', 'User Data') + ]; + return dirs.filter((d) => existsSync(d.dataDir)); +} + +export function browserDirs(home = homedir(), platform: NodeJS.Platform = process.platform): BrowserDir[] { + if (platform === 'win32') return windowsBrowserDirs(home); + return platform === 'linux' ? linuxBrowserDirs(home) : macBrowserDirs(home); +} + +/** + * The registry, through reg.exe (always present). A seam so tests on any + * platform can watch what Set up writes. + */ +export const windowsRegistry = { + read(key: string, value?: string): string | null { + try { + const out = execFileSync('reg', ['query', key, ...(value ? ['/v', value] : ['/ve'])], { encoding: 'utf8', timeout: 5000, windowsHide: true }); + return /REG_(?:EXPAND_)?SZ\s+(.*)$/m.exec(out)?.[1]?.trim() ?? null; + } catch { + // quiet: no such key. + return null; + } + }, + write(key: string, data: string): void { + execFileSync('reg', ['add', key, '/ve', '/t', 'REG_SZ', '/d', data, '/f'], { timeout: 5000, windowsHide: true, stdio: 'ignore' }); + } +}; + +/** The address of a browser's extensions page, by app name, bundle path or executable. */ export function extensionsPageUrl(app: string): string { if (/(^|\/)Arc(\.app)?$/.test(app)) return 'arc://extensions'; - if (/(^|\/)Microsoft Edge(\.app)?$/.test(app)) return 'edge://extensions'; + if (/(^|\/)Microsoft Edge(\.app)?$|microsoft-edge|msedge/.test(app)) return 'edge://extensions'; return 'chrome://extensions'; } -/** The bundle id of the user's default web browser, or null when it cannot be read. */ -export function defaultBrowserBundleId(home = homedir()): string | null { +/** The id of the user's default web browser (a bundle id, or a .desktop file), or null when it cannot be read. */ +export function defaultBrowserId(home = homedir(), platform: NodeJS.Platform = process.platform): string | null { + if (platform === 'win32') { + return windowsRegistry.read('HKCU\\Software\\Microsoft\\Windows\\Shell\\Associations\\UrlAssociations\\https\\UserChoice', 'ProgId'); + } + if (platform === 'linux') { + try { + return execFileSync('xdg-settings', ['get', 'default-web-browser'], { encoding: 'utf8', timeout: 2000 }).trim() || null; + } catch { + // quiet: no xdg-settings only costs the default browser its place at the top. + return null; + } + } const plist = join(home, 'Library', 'Preferences', 'com.apple.LaunchServices', 'com.apple.launchservices.secure.plist'); if (!existsSync(plist)) return null; try { @@ -170,22 +300,36 @@ export function defaultBrowserBundleId(home = homedir()): string | null { } } +/** Where `setupBrowsers` looks for a launcher: Applications folders on a Mac, PATH on Linux; Windows launchers are absolute. */ +function defaultAppDirs(home: string, platform: NodeJS.Platform): string[] { + if (platform === 'win32') return ['']; + if (platform === 'linux') return (process.env.PATH ?? '/usr/local/bin:/usr/bin:/bin').split(':').filter(Boolean); + return ['/Applications', join(home, 'Applications')]; +} + /** * The browsers the user can Load unpacked into, for Set up's steps: used on - * this Mac (a data folder) and still installed (an app bundle), the default - * browser first. Not Chrome for Testing, which only the dev harness drives. + * this computer (a data folder) and still installed (an app bundle, or a + * command on PATH), the default browser first. Not Chrome for Testing, which + * only the dev harness drives. */ export function setupBrowsers( home = homedir(), - appDirs = ['/Applications', join(home, 'Applications')], - defaultBundleId = defaultBrowserBundleId(home) + platform: NodeJS.Platform = process.platform, + appDirs = defaultAppDirs(home, platform), + defaultId = defaultBrowserId(home, platform) ): SetupBrowser[] { - const isDefault = (d: BrowserDir) => d.bundleId.toLowerCase() === defaultBundleId?.toLowerCase(); - return browserDirs(home) + const isDefault = (d: BrowserDir) => d.defaultIds.some((id) => id.toLowerCase() === defaultId?.toLowerCase()); + return browserDirs(home, platform) .filter((d) => d.name !== 'Google Chrome for Testing' && existsSync(d.dataDir)) - .filter((d) => appDirs.some((dir) => existsSync(join(dir, `${d.app}.app`)))) - .sort((a, b) => Number(isDefault(b)) - Number(isDefault(a))) - .map((d) => ({ name: d.name, app: d.app, extensionsUrl: extensionsPageUrl(d.app) })); + .flatMap((d) => { + const launcher = d.launchers.find((l) => appDirs.some((dir) => existsSync(join(dir, l)))); + // On Linux the command that exists is the one to run (on Windows, the + // executable); a Mac opens the bundle by name. + return launcher ? [{ d, app: platform === 'darwin' ? d.app : launcher }] : []; + }) + .sort((a, b) => Number(isDefault(b.d)) - Number(isDefault(a.d))) + .map(({ d, app }) => ({ name: d.name, app, extensionsUrl: extensionsPageUrl(app) })); } /** sh-quote: the paths go into a script, and "Application Support" has a space. */ @@ -203,6 +347,18 @@ export function wrapperScript(nodeCommand: string, hostScript: string, configPat ].join('\n'); } +/** The Windows wrapper: a batch file, since a manifest cannot pass environment or arguments. */ +export function wrapperBatch(nodeCommand: string, hostScript: string, configPath: string): string { + return [ + '@echo off', + 'rem Stem browser control: the native-messaging host the Stem extension talks to.', + 'rem Written by Stem (Settings → Features → Browser control → Set up); rewritten when Stem moves or updates.', + 'set ELECTRON_RUN_AS_NODE=1', + `"${nodeCommand}" "${hostScript}" "${configPath}" %*`, + '' + ].join('\r\n'); +} + export function hostManifest(wrapperPath: string): Record { return { name: BROWSER_NATIVE_HOST, @@ -259,9 +415,10 @@ async function syncExtension(src: InstallSource, paths: BrowserInstallPaths): Pr return true; } -async function writeWrapper(src: InstallSource, paths: BrowserInstallPaths): Promise { +async function writeWrapper(src: InstallSource, paths: BrowserInstallPaths, platform: NodeJS.Platform): Promise { await mkdir(paths.stateDir, { recursive: true, mode: 0o700 }); - await writeFile(paths.wrapperPath, wrapperScript(src.nodeCommand, src.hostScript, paths.configPath), { + const body = platform === 'win32' ? wrapperBatch : wrapperScript; + await writeFile(paths.wrapperPath, body(src.nodeCommand, src.hostScript, paths.configPath), { mode: 0o700 }); await chmod(paths.wrapperPath, 0o700).catch(() => undefined); @@ -271,14 +428,21 @@ async function writeWrapper(src: InstallSource, paths: BrowserInstallPaths): Pro export async function installBrowserControl( src: InstallSource, paths = browserInstallPaths(), - home = homedir() + home = homedir(), + platform: NodeJS.Platform = process.platform ): Promise { await ensureHostConfig(paths); const extensionChanged = await syncExtension(src, paths); - await writeWrapper(src, paths); + await writeWrapper(src, paths, platform); const manifest = `${JSON.stringify(hostManifest(paths.wrapperPath), null, 2)}\n`; const registered: string[] = []; - for (const d of browserDirs(home)) { + if (platform === 'win32') await writeFile(paths.manifestPath, manifest); + for (const d of browserDirs(home, platform)) { + if (d.registryKey) { + windowsRegistry.write(`${d.registryKey}\\${BROWSER_NATIVE_HOST}`, paths.manifestPath); + registered.push(d.name); + continue; + } await mkdir(d.hostsDir, { recursive: true }); await writeFile(join(d.hostsDir, `${BROWSER_NATIVE_HOST}.json`), manifest); registered.push(d.name); @@ -287,8 +451,16 @@ export async function installBrowserControl( } /** Whether some browser's manifest points at THIS profile's wrapper — i.e. Set up ran here last. */ -export function installedHere(paths = browserInstallPaths(), home = homedir()): boolean { - for (const d of browserDirs(home)) { +export function installedHere( + paths = browserInstallPaths(), + home = homedir(), + platform: NodeJS.Platform = process.platform +): boolean { + for (const d of browserDirs(home, platform)) { + if (d.registryKey) { + if (windowsRegistry.read(`${d.registryKey}\\${BROWSER_NATIVE_HOST}`)?.toLowerCase() === paths.manifestPath.toLowerCase()) return true; + continue; + } try { const m = JSON.parse(readFileSync(join(d.hostsDir, `${BROWSER_NATIVE_HOST}.json`), 'utf8')) as { path?: unknown }; if (m.path === paths.wrapperPath) return true; @@ -303,14 +475,21 @@ export function installedHere(paths = browserInstallPaths(), home = homedir()): * At startup: when Set up ran for this profile, bring the copy up to date — * the extension after a Stem update, the wrapper after the app moved. Never * registers anything new; that is Set up's job. + * + * An AppImage moves on every launch: its binary runs from a fresh mount that + * goes away when Stem quits. The wrapper written here points at this launch's + * mount, so while Stem is closed the browser cannot start the host, and the + * extension's 30 s reconnect alarm picks Stem up again once it is back. */ export async function refreshBrowserControl( src: InstallSource, paths = browserInstallPaths(), - home = homedir() + home = homedir(), + platform: NodeJS.Platform = process.platform ): Promise<{ extensionChanged: boolean } | null> { - if (!installedHere(paths, home)) return null; + if (!installedHere(paths, home, platform)) return null; const extensionChanged = await syncExtension(src, paths); - await writeWrapper(src, paths); + await writeWrapper(src, paths, platform); + if (platform === 'win32') await writeFile(paths.manifestPath, `${JSON.stringify(hostManifest(paths.wrapperPath), null, 2)}\n`); return { extensionChanged }; } diff --git a/src/desktop/browser-host/native-host-main.ts b/src/desktop/browser-host/native-host-main.ts index 763040ad..e10c55e7 100644 --- a/src/desktop/browser-host/native-host-main.ts +++ b/src/desktop/browser-host/native-host-main.ts @@ -8,15 +8,18 @@ // Stem app's unix socket, frames from the socket go to the extension (stdout). // Both legs use Chrome's framing, so a frame passes through unparsed — except: // -// - The first frame on every socket connection is this host's own -// `host-hello`: the token from the config Set up wrote (0600, this profile's -// state folder) and the browser that launched it, read off the parent process. -// Proof that the peer was installed by this Stem — not proof against malware -// running as the same user, which can read the same file. +// - Every socket connection opens with a handshake (handshake.ts): this host's +// `host-hello` names the browser that launched it, read off the parent +// process, and each side proves it holds the token from the config Set up +// wrote (0600, this profile's state folder) without sending it. Nothing is +// relayed until the desktop has proved itself. Proof that both were installed +// by this Stem — not proof against malware running as the same user, which +// can read the same file. // - A result that carries finished downloads has each file copied out of the // user's Downloads into Stem's spool first. This process is the browser's // child, so macOS attributes that read to the browser, which already has // Downloads; the Stem app reading it directly would raise a privacy prompt. +// (Linux has no such prompt; the copy is kept so both take one path.) // // It never exits on its own while the browser keeps the port open: with Stem // not running it retries the socket every two seconds, and that open port is @@ -26,17 +29,21 @@ import { execFileSync } from 'node:child_process'; import { randomBytes } from 'node:crypto'; -import { readFileSync } from 'node:fs'; +import { readFileSync, readlinkSync } from 'node:fs'; import { copyFile, mkdir } from 'node:fs/promises'; import { createConnection, type Socket } from 'node:net'; -import { basename, join } from 'node:path'; +import { basename, join, win32 } from 'node:path'; +import { handshakeProof, isNonce, newNonce, proofMatches } from './handshake'; import { encodeFrame, FrameReader, MAX_TO_EXTENSION_BYTES, type ExtensionDownload, type FromExtension, - type HostHello + type HostHello, + appImageName, + linuxBrowserName, + windowsBrowserName } from '../../shared/browser-native'; interface HostConfig { @@ -47,16 +54,67 @@ interface HostConfig { const RETRY_MS = 2000; -/** The browser's app bundle and name, from the process that started us. */ +/** + * The browser and its name, from the process that started us: a Mac reports + * the binary inside the bundle and the bundle is the browser; on Linux the + * executable is the browser (and what launching it again runs). + */ export function browserOf(command: string): { appPath: string; appName: string } { const at = command.indexOf('.app/'); - const appPath = at >= 0 ? command.slice(0, at + 4) : command; - const appName = basename(appPath).replace(/\.app$/, '') || 'Browser'; - return { appPath, appName }; + if (at >= 0 || command.endsWith('.app')) { + const appPath = at >= 0 ? command.slice(0, at + 4) : command; + return { appPath, appName: basename(appPath).replace(/\.app$/, '') || 'Browser' }; + } + if (/^[a-z]:\\/i.test(command)) { + return { appPath: command, appName: windowsBrowserName(command) ?? (win32.basename(command).replace(/\.exe$/i, '') || 'Browser') }; + } + const appName = linuxBrowserName(command) ?? (/\.appimage$/i.test(command) ? appImageName(command) : basename(command) || 'Browser'); + return { appPath: command, appName }; +} + +/** + * Linux: the AppImage a browser runs from, by the environment it passed us + * (the AppImage runtime sets APPIMAGE and APPDIR; the browser's own + * /proc//environ is no help, Chromium writes its process title over it). + * The executable sits in a mount that changes on every launch, so the + * AppImage file is what stays the same and what starts the browser again. + * The executable must be inside APPDIR: a browser started from another + * AppImage (Stem's own) inherits its variables. + */ +export function appImageOf(exe: string, env: NodeJS.ProcessEnv): string | null { + const image = env.APPIMAGE; + const dir = env.APPDIR?.replace(/\/+$/, ''); + if (!image?.startsWith('/') || !dir?.startsWith('/') || !exe.startsWith(`${dir}/`)) return null; + return image; } +/** + * Windows: the browser starts a batch-file host through cmd.exe, so the + * browser is the first ancestor that is not cmd.exe. One PowerShell call walks + * the chain (Node has no API for another process's executable). + */ +const WINDOWS_PARENT = [ + '$p = Get-CimInstance Win32_Process -Filter "ProcessId=$args";', + 'while ($p -and $p.Name -ieq "cmd.exe") { $p = Get-CimInstance Win32_Process -Filter "ProcessId=$($p.ParentProcessId)" }', + 'if ($p) { $p.ExecutablePath }' +].join(' '); + function parentCommand(): string { try { + if (process.platform === 'win32') { + const out = execFileSync( + 'powershell.exe', + ['-NoProfile', '-NonInteractive', '-Command', `& { ${WINDOWS_PARENT} } ${process.ppid}`], + { encoding: 'utf8', timeout: 10_000, windowsHide: true } + ).trim(); + return out || 'Browser'; + } + if (process.platform === 'linux') { + // `ps -o comm=` is cut to 15 characters there; the exe link is the whole + // path. A browser updated in place while running reads "… (deleted)". + const exe = readlinkSync(`/proc/${process.ppid}/exe`).replace(/ \(deleted\)$/, ''); + return appImageOf(exe, process.env) ?? exe; + } return execFileSync('/bin/ps', ['-o', 'comm=', '-p', String(process.ppid)], { encoding: 'utf8' }).trim(); } catch { return 'Browser'; @@ -88,6 +146,17 @@ async function spoolDownloads(downloads: ExtensionDownload[], spoolDir: string): return out; } +/** Is this frame the desktop's server-hello, with a proof under the token for our nonce? */ +export function serverProved(token: string, nonce: string, body: Buffer): boolean { + let m: { type?: unknown; nonce?: unknown; proof?: unknown }; + try { + m = JSON.parse(body.toString('utf8')) as typeof m; + } catch { + return false; + } + return m?.type === 'server-hello' && isNonce(m.nonce) && proofMatches(token, 'stem', nonce, m.proof); +} + function main(): void { const config = JSON.parse(readFileSync(process.argv[2] ?? '', 'utf8')) as HostConfig; const browser = browserOf(parentCommand()); @@ -115,15 +184,29 @@ function main(): void { const s = createConnection(config.socketPath); socket = s; const reader = new FrameReader(MAX_TO_EXTENSION_BYTES); + const nonce = newNonce(); + /** Set once the desktop proved it holds the token; until then nothing is relayed either way. */ + let verified = false; s.on('connect', () => { - const first: HostHello = { type: 'host-hello', token: config.token, ...browser }; + const first: HostHello = { type: 'host-hello', nonce, ...browser }; s.write(encodeFrame(first)); - if (hello) s.write(hello); - status(true); }); s.on('data', (chunk: Buffer) => { try { for (const body of reader.push(chunk)) { + if (!verified) { + if (!serverProved(config.token, nonce, body)) { + log('a peer on the socket could not prove it is Stem; not relaying to it'); + s.destroy(); + return; + } + const answer = JSON.parse(body.toString('utf8')) as { nonce: string }; + s.write(encodeFrame({ type: 'host-proof', proof: handshakeProof(config.token, 'host', answer.nonce) })); + verified = true; + if (hello) s.write(hello); + status(true); + continue; + } const head = Buffer.alloc(4); head.writeUInt32LE(body.length, 0); toExtension(Buffer.concat([head, body])); diff --git a/src/desktop/browser-host/store.ts b/src/desktop/browser-host/store.ts index c0cc9212..3bc129ea 100644 --- a/src/desktop/browser-host/store.ts +++ b/src/desktop/browser-host/store.ts @@ -9,9 +9,25 @@ import { log } from '../../server/log'; // decision. Off by default; only a window on this machine can flip it. export interface KnownBrowser { - /** The app bundle the native host was launched from ("/Applications/Arc.app"). */ + /** + * The app bundle the native host was launched from, then "#" and the + * profile id the extension keeps ("/Applications/Arc.app#3f2a…"). Entries + * written before profiles were told apart are the bare bundle path; the + * first profile of that app to connect takes the entry over. + */ id: string; + /** The app's display name ("Arc"). */ name: string; + /** What the user called this profile ("Work"). */ + label?: string; + /** A few sites open in it when it last connected. */ + hint?: string; +} + +/** The app bundle a known browser is launched from. */ +export function appOf(id: string): string { + const at = id.indexOf('#'); + return at >= 0 ? id.slice(0, at) : id; } export interface BrowserHostSettings { @@ -37,7 +53,12 @@ export async function readBrowserHostSettings(): Promise { const known = Array.isArray(parsed?.known) ? parsed.known.filter( (b): b is KnownBrowser => !!b && typeof b.id === 'string' && !!b.id && typeof b.name === 'string' - ) + ).map((b) => ({ + id: b.id, + name: b.name, + ...(typeof b.label === 'string' && b.label.trim() ? { label: b.label.trim() } : {}), + ...(typeof b.hint === 'string' && b.hint ? { hint: b.hint } : {}) + })) : []; return { enabled: parsed?.enabled === true, @@ -75,7 +96,7 @@ export async function writeBrowserHostEnabled(enabled: boolean): Promise ({ ...cur, enabled })); log( 'browser-host', - enabled ? 'this Mac now lets Stem drive its browser' : 'this Mac stopped letting Stem drive its browser' + enabled ? 'this computer now lets Stem drive its browser' : 'this computer stopped letting Stem drive its browser' ); return next; } diff --git a/src/desktop/client-store.ts b/src/desktop/client-store.ts index bc9118df..69ef5b52 100644 --- a/src/desktop/client-store.ts +++ b/src/desktop/client-store.ts @@ -1,8 +1,9 @@ -import { chmod, mkdir, readFile, rename, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { host } from '../server/host'; import { log } from '../server/log'; import type { ClientSettings } from '../shared/types'; +import { replaceFile } from '../server/replace-file'; // What this CLIENT knows about itself: which server it talks to, its row in that // server's device registry, the bearer token that proves it, and the handful of @@ -93,7 +94,7 @@ export function updateClientDocument(mutate: (doc: StoredClient) => T | Promi encoding: 'utf8', mode: 0o600 }); - await rename(tmp, path); + await replaceFile(tmp, path); // The rename carries the tmp file's 0600 along, but an existing file replaced // on Windows can differ; re-assert either way. await chmod(path, 0o600).catch(() => undefined); diff --git a/src/desktop/computer-host/banner.ts b/src/desktop/computer-host/banner.ts index d8b65cd1..dcb8dd41 100644 --- a/src/desktop/computer-host/banner.ts +++ b/src/desktop/computer-host/banner.ts @@ -1,5 +1,6 @@ import { BrowserWindow, screen } from 'electron'; -import { workspaceVisibilityOptions } from '../platform'; +import { isWaylandSession, workspaceVisibilityOptions } from '../platform'; +import { placeOwnWindow } from './helper'; // The "Stem is controlling this Mac" pill: a small always-on-top window at the // top centre of the main display for as long as a run is on. It is the one @@ -16,16 +17,18 @@ import { workspaceVisibilityOptions } from '../platform'; const WIDTH = 340; const HEIGHT = 44; +/** The window's title: how the Linux helper finds it to place it (Wayland ignores setPosition). */ +const TITLE = 'Stem computer control'; -const HTML = `
Stem is controlling this Mac +
${labelFor(null)}
`; // An https URL, not a custom scheme: Chromium routes unknown schemes to the OS @@ -43,7 +46,7 @@ export interface ComputerBanner { } function labelFor(app: string | null): string { - return app ? `Stem is controlling ${app}` : 'Stem is controlling this Mac'; + return app ? `Stem is controlling ${app}` : `Stem is controlling this ${process.platform === 'darwin' ? 'Mac' : 'computer'}`; } export function createComputerBanner(): ComputerBanner { @@ -59,6 +62,7 @@ export function createComputerBanner(): ComputerBanner { function ensure(): BrowserWindow { if (win && !win.isDestroyed()) return win; win = new BrowserWindow({ + title: TITLE, width: WIDTH, height: HEIGHT, frame: false, @@ -103,8 +107,12 @@ export function createComputerBanner(): ComputerBanner { show() { const w = ensure(); const area = screen.getPrimaryDisplay().workArea; - w.setPosition(Math.round(area.x + (area.width - WIDTH) / 2), area.y + 8); + const x = Math.round(area.x + (area.width - WIDTH) / 2); + w.setPosition(x, area.y + 8); w.showInactive(); + // A Wayland compositor places new windows itself (GNOME: centred), so + // Stem's GNOME extension moves the pill to the top, above everything. + if (isWaylandSession()) placeOwnWindow(TITLE, x, area.y + 8); }, hide() { if (win && !win.isDestroyed()) win.hide(); diff --git a/src/desktop/computer-host/gnome-extension.ts b/src/desktop/computer-host/gnome-extension.ts new file mode 100644 index 00000000..9c9bcbaa --- /dev/null +++ b/src/desktop/computer-host/gnome-extension.ts @@ -0,0 +1,140 @@ +import { execFile } from 'node:child_process'; +import { cp, readFile, rm } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import type { LinuxComputerSetup } from '../../shared/types'; +import { linuxNativeRoot, oneShot } from './helper'; + +// Stem's GNOME Shell extension (native/linux/gnome-extension), which the Linux +// helper asks for the screen, the window list and a virtual pointer and +// keyboard: Wayland gives an app none of those. Installing is a copy into the +// person's own extensions folder plus adding it to GNOME's enabled list; the +// shell only discovers a new extension when it starts, so a first install (and +// an update to a newer copy) takes effect at the next login — the settings row +// says so. Nothing here needs root. + +export const GNOME_EXTENSION_UUID = 'stem-computer@awantech.sk'; + +const run = promisify(execFile); + +export interface GnomeExtensionDeps { + /** The bundled copy; defaults to resources/linux (packaged) or native/linux. */ + bundledDir?: string; + /** $XDG_DATA_HOME, else ~/.local/share. */ + dataHome?: string; + gsettings?(args: string[]): Promise; + /** Ask the running extension to import the new files; defaults to the helper's reload-extension. */ + reload?(): Promise; +} + +/** The running extension re-imports core.js (needs its loader); false when it cannot. */ +async function reloadRunning(): Promise { + const reply = await oneShot('reload-extension'); + return reply.ok; +} + +function paths(deps: GnomeExtensionDeps) { + const dataHome = deps.dataHome ?? (process.env.XDG_DATA_HOME || join(homedir(), '.local', 'share')); + return { + bundled: deps.bundledDir ?? join(linuxNativeRoot(), 'gnome-extension', GNOME_EXTENSION_UUID), + installed: join(dataHome, 'gnome-shell', 'extensions', GNOME_EXTENSION_UUID) + }; +} + +async function gsettingsCli(args: string[]): Promise { + const { stdout } = await run('gsettings', args, { timeout: 5000 }); + return stdout.trim(); +} + +/** The `version` in a copy's metadata.json, or null when there is no readable copy. */ +async function versionAt(dir: string): Promise { + try { + const meta = JSON.parse(await readFile(join(dir, 'metadata.json'), 'utf8')) as { version?: unknown }; + return typeof meta.version === 'number' ? meta.version : null; + } catch { + return null; + } +} + +/** Is the copy in the person's extensions folder the one this Stem ships? */ +export async function gnomeExtensionCurrent(deps: GnomeExtensionDeps = {}): Promise { + const p = paths(deps); + const [have, ship] = await Promise.all([versionAt(p.installed), versionAt(p.bundled)]); + return have !== null && have === ship; +} + +/** A GVariant string list as gsettings prints it: `['a', 'b']` or `@as []`. */ +export function parseStrv(text: string): string[] { + return [...text.matchAll(/'((?:[^'\\]|\\.)*)'/g)].map((m) => m[1]!); +} + +function formatStrv(list: string[]): string { + return `[${list.map((s) => `'${s.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`).join(', ')}]`; +} + +/** + * Copies the bundled extension in (replacing an older copy) and puts it on + * GNOME's enabled list. A running shell that already knows it enables it at + * once; otherwise it loads at the next login. + */ +export async function installGnomeExtension(deps: GnomeExtensionDeps = {}): Promise { + const p = paths(deps); + const gsettings = deps.gsettings ?? gsettingsCli; + if (!(await gnomeExtensionCurrent(deps))) { + if ((await versionAt(p.bundled)) === null) throw new Error(`The GNOME extension is missing from this Stem (looked in ${p.bundled}).`); + await rm(p.installed, { recursive: true, force: true }); + await cp(p.bundled, p.installed, { recursive: true }); + // A running copy with the loader takes the new core.js at once. + await (deps.reload ?? reloadRunning)().catch(() => false); + } + const enabled = parseStrv(await gsettings(['get', 'org.gnome.shell', 'enabled-extensions'])); + if (!enabled.includes(GNOME_EXTENSION_UUID)) { + await gsettings(['set', 'org.gnome.shell', 'enabled-extensions', formatStrv([...enabled, GNOME_EXTENSION_UUID])]); + } + const disabled = parseStrv(await gsettings(['get', 'org.gnome.shell', 'disabled-extensions']).catch(() => '[]')); + if (disabled.includes(GNOME_EXTENSION_UUID)) { + await gsettings(['set', 'org.gnome.shell', 'disabled-extensions', formatStrv(disabled.filter((u) => u !== GNOME_EXTENSION_UUID))]); + } +} + +/** + * A running extension that still speaks the helper's protocol, but older + * files than this Stem ships: the newer copy goes in quietly and takes over at + * the next login. Never touches the enabled list. + */ +export async function refreshGnomeExtensionFiles(deps: GnomeExtensionDeps = {}): Promise { + if (await gnomeExtensionCurrent(deps)) return false; + const p = paths(deps); + if ((await versionAt(p.bundled)) === null) return false; + await rm(p.installed, { recursive: true, force: true }); + await cp(p.bundled, p.installed, { recursive: true }); + await (deps.reload ?? reloadRunning)().catch(() => false); + return true; +} + +/** + * The setup as the settings row shows it, from what the helper saw running + * (`state`) and what is on disk. A shell that has not loaded the extension + * needs an install when the copy on disk is missing or older, and only a new + * login when the current copy is already there. + */ +export async function linuxSetupFrom( + helper: { state: string; error?: string | null } | undefined, + deps: GnomeExtensionDeps = {} +): Promise { + const state = helper?.state ?? 'not-loaded'; + switch (state) { + case 'ready': + case 'disabled': + case 'extensions-off': + case 'not-gnome': + case 'not-wayland': + return { state }; + case 'error': + return { state: 'error', ...(helper?.error ? { error: helper.error } : {}) }; + default: + // 'not-loaded' or 'outdated': the shell runs no current copy. + return { state: (await gnomeExtensionCurrent(deps)) ? 'relogin' : 'not-installed' }; + } +} diff --git a/src/desktop/computer-host/helper.ts b/src/desktop/computer-host/helper.ts index e96ae1e3..da16e85f 100644 --- a/src/desktop/computer-host/helper.ts +++ b/src/desktop/computer-host/helper.ts @@ -2,18 +2,22 @@ import { spawn, type ChildProcess } from 'node:child_process'; import { createInterface } from 'node:readline'; import { createHash } from 'node:crypto'; import { access, mkdir, readdir, readFile } from 'node:fs/promises'; -import { constants } from 'node:fs'; +import { constants, existsSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { host } from '../../server/host'; import { log } from '../../server/log'; import type { ComputerAccess, ComputerTarget, RecordedStep } from '../../shared/types'; -// The Swift helper (native/mac/stem-computer) as a child process: one JSON -// line in, one out, plus the unsolicited human-input event while a run is -// watched. Packaged, the binary ships at Resources/bin/stem-computer (see -// electron-builder.yml extraResources); in development it is compiled on -// demand into the profile's state folder so a fresh checkout works without -// remembering a build step. +// The helper as a child process: one JSON line in, one out, plus the +// unsolicited human-input event while a run is watched. On a Mac it is the +// Swift helper (native/mac/stem-computer), on Windows its C# twin +// (native/windows/stem-computer), on Linux a gjs script +// (native/linux/stem-computer) that reaches GNOME through Stem's Shell +// extension — all speaking the same lines. Packaged, the binary ships at +// resources/bin/stem-computer(.exe) and the Linux scripts under +// resources/linux (see electron-builder.yml extraResources); in development +// the Mac and Windows ones are compiled on demand into the profile's state +// folder so a fresh checkout works without remembering a build step. const CALL_TIMEOUT_MS = 30_000; @@ -38,6 +42,8 @@ export interface HelperReply { current?: string; /** dictate-start: the language the model settled on. */ locale?: string; + /** Linux status: the GNOME Shell extension as the helper sees it running. */ + linux?: { state: string; error?: string | null; protocol?: number; loader?: number }; } export type HelperEvent = @@ -54,7 +60,9 @@ export type HelperEvent = /** Where the helper binary is, building it first in development. Throws with a readable reason. */ export async function resolveHelperPath(): Promise { - if (process.platform !== 'darwin') throw new Error('Computer control is only available on macOS.'); + if (process.platform === 'win32') return resolveWindowsHelper(); + if (process.platform === 'linux') return resolveLinuxHelper(); + if (process.platform !== 'darwin') throw new Error('Computer control is only available on macOS, Windows and Linux.'); const packaged = join(process.resourcesPath ?? '', 'bin', 'stem-computer'); if (process.resourcesPath && (await exists(packaged))) return packaged; // Development: run the build script (the same one electron-builder's @@ -87,9 +95,77 @@ export async function resolveHelperPath(): Promise { return output; } -async function sourceHash(dir: string): Promise { +/** Windows: the packaged exe, or a development build with the C# compiler Windows ships. */ +async function resolveWindowsHelper(): Promise { + const packaged = join(process.resourcesPath ?? '', 'bin', 'stem-computer.exe'); + if (process.resourcesPath && (await exists(packaged, constants.R_OK))) return packaged; + const repo = resolve(__dirname, '..', '..'); + const script = join(repo, 'scripts', 'build-windows-helper.mjs'); + const sources = join(repo, 'native', 'windows', 'stem-computer'); + if (!(await exists(script, constants.R_OK))) { + throw new Error(`The computer-control helper is missing (looked in ${packaged}).`); + } + const output = join(host().stateRoot(), 'native', `stem-computer-${await sourceHash(sources, '.cs')}.exe`); + if (await exists(output, constants.R_OK)) return output; + await mkdir(dirname(output), { recursive: true }); + log('computer-host', 'building the helper for development', { output }); + const { command, env } = host().nodeSpawn(); + await new Promise((done, fail) => { + const child = spawn(command, [script, '--output', output], { + env: { ...process.env, ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true + }); + let err = ''; + child.stdout?.on('data', (c: Buffer) => (err += c.toString('utf8'))); + child.stderr?.on('data', (c: Buffer) => (err += c.toString('utf8'))); + child.on('error', fail); + child.on('exit', (code) => (code === 0 ? done() : fail(new Error(`Building the helper failed: ${err.trim().slice(-400)}`)))); + }); + return output; +} + +/** The Linux helper's folder: native/linux, packaged as resources/linux. */ +export function linuxNativeRoot(): string { + const packaged = join(process.resourcesPath ?? '', 'linux'); + if (process.resourcesPath && existsSync(join(packaged, 'stem-computer', 'main.js'))) return packaged; + return resolve(__dirname, '..', '..', 'native', 'linux'); +} + +/** Linux: a gjs script, nothing to build. gjs comes with GNOME Shell, the only desktop it serves. */ +async function resolveLinuxHelper(): Promise { + const main = join(linuxNativeRoot(), 'stem-computer', 'main.js'); + if (!(await exists(main, constants.R_OK))) throw new Error(`The computer-control helper is missing (looked in ${main}).`); + if (!(await gjsPath())) throw new GjsMissingError(); + return main; +} + +/** No gjs on PATH: not a GNOME desktop, or a stripped-down one. */ +export class GjsMissingError extends Error { + constructor() { + super('Computer control on Linux needs GNOME Shell (its gjs runtime was not found).'); + } +} + +let gjs: string | null | undefined; + +/** Where gjs is, looked up once. */ +export async function gjsPath(): Promise { + if (gjs !== undefined) return gjs; + for (const dir of (process.env.PATH ?? '').split(':').concat('/usr/bin', '/usr/local/bin')) { + if (dir && (await exists(join(dir, 'gjs')))) return (gjs = join(dir, 'gjs')); + } + return (gjs = null); +} + +/** How a helper is started: the binary itself, or gjs running the Linux script. */ +export function helperCommand(binary: string): { command: string; args: string[] } { + return binary.endsWith('.js') ? { command: gjs ?? 'gjs', args: ['-m', binary] } : { command: binary, args: [] }; +} + +async function sourceHash(dir: string, ext = '.swift'): Promise { const h = createHash('sha256'); - for (const name of (await readdir(dir)).filter((f) => f.endsWith('.swift')).sort()) { + for (const name of (await readdir(dir)).filter((f) => f.endsWith(ext)).sort()) { h.update(await readFile(join(dir, name))); } return h.digest('hex').slice(0, 16); @@ -118,7 +194,13 @@ export class HelperProcess { private readonly listeners = new Set<(event: HelperEvent) => void>(); constructor(binary: string) { - this.proc = spawn(binary, [], { stdio: ['pipe', 'pipe', 'pipe'] }); + // STEM_PARENT_PID: the Windows and Linux helpers leave Stem's own windows out of their lists. + const { command, args } = helperCommand(binary); + this.proc = spawn(command, args, { + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + env: { ...process.env, STEM_PARENT_PID: String(process.pid) } + }); const lines = createInterface({ input: this.proc.stdout! }); lines.on('line', (line) => this.onLine(line)); this.proc.stderr?.on('data', (chunk: Buffer) => { @@ -220,3 +302,16 @@ export async function oneShot(cmd: string, fields: Record = {}) proc.kill(); } } + +/** + * Move one of Stem's own windows (found by title) to (x, y) on a Wayland + * session, where the compositor places new windows itself (GNOME: centred) + * and ignores setPosition. Needs Stem's GNOME extension; without it the window + * stays where GNOME put it. Never throws. + */ +export function placeOwnWindow(title: string, x: number, y: number): void { + void oneShot('place-window', { title, x, y }).then( + (r) => r.ok || log('computer-host', 'could not place a window', { title, error: r.error }), + (e: unknown) => log('computer-host', 'could not place a window', { title, error: e instanceof Error ? e.message : String(e) }) + ); +} diff --git a/src/desktop/computer-host/index.ts b/src/desktop/computer-host/index.ts index 0a550199..8497dd3d 100644 --- a/src/desktop/computer-host/index.ts +++ b/src/desktop/computer-host/index.ts @@ -5,9 +5,11 @@ import type { ComputerHostLocalState, ComputerTarget, DeviceComputerRequest, - DeviceComputerResult + DeviceComputerResult, + LinuxComputerSetup } from '../../shared/types'; -import { HelperProcess, oneShot, resolveHelperPath, type HelperEvent, type HelperReply } from './helper'; +import { installGnomeExtension, linuxSetupFrom, refreshGnomeExtensionFiles } from './gnome-extension'; +import { GjsMissingError, HelperProcess, oneShot, resolveHelperPath, type HelperEvent, type HelperReply } from './helper'; import { readComputerHostEnabled, writeComputerHostEnabled } from './store'; // The client half of the `computer` tool: THIS Mac, performing the screen @@ -54,13 +56,28 @@ export interface ComputerHostDeps { /** The person pressed Stop on the banner. */ onStop?(handler: () => void): void; }; + /** Tests pretend to be another platform. */ + platform?: NodeJS.Platform; /** How a helper is started and asked one-off questions; defaults to the real binary. */ helpers?: { spawn(): Promise; oneShot(cmd: string): Promise; }; + /** Linux: the GNOME extension's install and on-disk state; defaults to the real ones. */ + gnome?: { + install(): Promise; + setup(helper: HelperReply['linux']): Promise; + /** Copy in newer files under a running, compatible extension (they load at the next login). */ + refresh?(): Promise; + }; } +const realGnome: NonNullable = { + install: () => installGnomeExtension(), + setup: (helper) => linuxSetupFrom(helper), + refresh: () => refreshGnomeExtensionFiles() +}; + const realHelpers: NonNullable = { spawn: async () => new HelperProcess(await resolveHelperPath()), oneShot: (cmd) => oneShot(cmd) @@ -75,7 +92,10 @@ export interface ComputerHost { onEnd(end: { threadId: string }): void; localState(): Promise; setEnabled(enabled: boolean): Promise; - /** Ask macOS for the missing grants (prompts appear); answers with the state afterwards. */ + /** + * Ask macOS for the missing grants (prompts appear), or on Linux install and + * enable the GNOME extension; answers with the state afterwards. + */ requestAccess(): Promise; /** Quit: end any run, kill the helper. */ close(): void; @@ -93,6 +113,8 @@ interface Run { aborted: boolean; /** The window the run works on; null = the whole screen (pointer + kill switch). */ target: ComputerTarget | null; + /** The platform note has gone out on this run's first answer. */ + noted: boolean; } /** The macOS major version (Darwin 23 = macOS 14), or 0 off macOS / when unreadable. */ @@ -102,23 +124,67 @@ function macOSMajor(): number { return Number.isFinite(darwin) ? darwin - 9 : 0; } +/** + * What the model is told once per run on a computer that is not a Mac: the + * tool's habits (cmd for shortcuts, Spaces) come from macOS. + */ +const PLATFORM_NOTES: Partial> = { + win32: + 'This computer runs Windows. Shortcuts use ctrl where a Mac uses cmd (cmd is sent as ctrl; win is the Windows key). ' + + 'Windows have no Spaces: list_windows says "other desktop" for a window on another virtual desktop, and a minimized ' + + 'window cannot be pictured until it is restored. In window mode a background window takes plain keys and typing, ' + + 'but no ctrl, alt or shift combinations (shift+arrow selection too): use its menus through snapshot and press, or the whole screen.', + linux: + 'This computer runs Linux (GNOME on Wayland). Shortcuts use ctrl where a Mac uses cmd (cmd is sent as ctrl; super is ' + + 'the Super key). Workspaces stand in for Spaces: list_windows says "other workspace", and a minimized window cannot be ' + + 'pictured until it is restored. In window mode a background window takes no keys at all (Wayland sends keys only to ' + + 'the window in front): type text with `type` (it goes into the field you last clicked or focused), and use snapshot ' + + 'and press for buttons; key presses other than Return need the whole screen. GTK 4 apps may report control positions ' + + 'a little off, so in window mode prefer snapshot ids to coordinates.' +}; + export function createComputerHost(deps: ComputerHostDeps): ComputerHost { // macOS 14 or newer: per-window capture (ScreenCaptureKit's screenshot API) is // what window mode is built on, and the helper is compiled against it. - const supported = process.platform === 'darwin' && macOSMajor() >= 14; + // Windows: PrintWindow and UI Automation, in every Windows 10 and 11. + // Linux: GNOME on Wayland through Stem's Shell extension; whether this + // desktop is one is the extension setup's to say (`linux` below). + const platform = deps.platform ?? process.platform; + const supported = (platform === 'darwin' && macOSMajor() >= 14) || platform === 'win32' || platform === 'linux'; + const here = platform === 'darwin' ? 'Mac' : 'computer'; + const platformNote = PLATFORM_NOTES[platform]; const helpers = deps.helpers ?? realHelpers; + const gnome = deps.gnome ?? realGnome; let run: Run | null = null; let lastAccess: ComputerAccess | null = null; + let linux: LinuxComputerSetup | undefined; deps.banner.onStop?.(() => { if (run) stoppedFromBanner(run); }); - async function checkAccess(): Promise { + async function checkAccess(cmd: 'status' | 'request-access' = 'status'): Promise { if (!supported) return null; try { - const reply = await helpers.oneShot('status'); + const reply = await helpers.oneShot(cmd); if (reply.ok && reply.status) lastAccess = reply.status; + if (platform === 'linux') { + // An outdated extension that has the loader is updated in place: new + // files, a reload, and it is current — no button, no login. + if (!run && reply.ok && reply.linux?.state === 'outdated' && (reply.linux.loader ?? 0) >= 1 && cmd === 'status') { + await gnome.install().catch(() => undefined); + return checkAccess('request-access'); + } + linux = await gnome.setup(reply.ok ? reply.linux : undefined); + // Never swap the extension's code under a running run. + if (!run && linux.state === 'ready' && (await gnome.refresh?.().catch(() => false))) { + log('computer-host', 'copied a newer GNOME extension in; it loads at the next login'); + } + } } catch (e) { + if (platform === 'linux' && e instanceof GjsMissingError) { + linux = { state: 'no-gjs' }; + lastAccess = { screen: false, accessibility: false, inputMonitoring: false }; + } log('computer-host', 'could not read the helper status', { error: e instanceof Error ? e.message : String(e) }); @@ -131,7 +197,9 @@ export function createComputerHost(deps: ComputerHostDeps): ComputerHost { const enabled = await readComputerHostEnabled(); const access = enabled ? await checkAccess() : lastAccess; await deps - .invoke('computerHost:announce', [{ enabled, platform: 'darwin', ...(access ? { access } : {}) }]) + .invoke('computerHost:announce', [ + { enabled, platform: platform === 'win32' || platform === 'linux' ? platform : 'darwin', ...(access ? { access } : {}) } + ]) .catch((e) => { // An older server has no such channel; this Mac simply cannot be a target there. log('computer-host', 'could not announce', { @@ -141,7 +209,7 @@ export function createComputerHost(deps: ComputerHostDeps): ComputerHost { } function state(): ComputerHostLocalState { - return { supported, enabled: false, access: lastAccess }; + return { supported, enabled: false, access: lastAccess, ...(linux ? { linux } : {}) }; } function touch(r: Run): void { @@ -198,7 +266,8 @@ export function createComputerHost(deps: ComputerHostDeps): ComputerHost { idle: setTimeout(() => undefined, 0), inFlight: null, aborted: false, - target: null + target: null, + noted: false }; helper.onEvent((event) => { if (event.event === 'human-input') humanTookOver(r, event.kind); @@ -318,23 +387,23 @@ export function createComputerHost(deps: ComputerHostDeps): ComputerHost { if (!supported) return { ok: false, - error: 'Computer control is only available on macOS.' + error: 'Computer control is only available on macOS 14 or newer, Windows, and Linux with GNOME on Wayland.' }; // The gate, read fresh from this disk on every request. if (!(await readComputerHostEnabled())) { return { ok: false, error: - 'This Mac does not let Stem control its screen. The switch is in Settings → Features → Computer control, ' + + `This ${here} does not let Stem control its screen. The switch is in Settings → Features → Computer control, ` + 'on that computer.' }; } - // One run at a time, per Mac: a second thread's action while another + // One run at a time, per computer: a second thread's action while another // thread is driving would fight it for the one cursor there is. if (run && run.threadId !== request.threadId) { return { ok: false, - error: 'Another conversation is controlling this Mac right now; try again when it is done.' + error: `Another conversation is controlling this ${here} right now; try again when it is done.` }; } let r = run; @@ -361,8 +430,12 @@ export function createComputerHost(deps: ComputerHostDeps): ComputerHost { }; r!.inFlight = { requestId: request.requestId, fail: done }; void r!.helper.call(cmd, fields, timeout).then((reply) => { - const result = fromReply(reply); + let result = fromReply(reply); noteTarget(r!, result); + if (platformNote && result.ok && !r!.noted) { + r!.noted = true; + result = { ...result, text: result.text ? `${platformNote}\n\n${result.text}` : platformNote }; + } done(result); }); }); @@ -398,11 +471,11 @@ export function createComputerHost(deps: ComputerHostDeps): ComputerHost { if (run && run.threadId === threadId) endRun(run, 'turn-ended'); }, - localState: async () => ({ - ...state(), - enabled: await readComputerHostEnabled(), - access: await checkAccess() - }), + localState: async () => { + const enabled = await readComputerHostEnabled(); + const access = await checkAccess(); + return { ...state(), enabled, access }; + }, async setEnabled(enabled) { if (!supported) return state(); @@ -414,14 +487,18 @@ export function createComputerHost(deps: ComputerHostDeps): ComputerHost { async requestAccess() { if (!supported) return state(); - try { - const reply = await helpers.oneShot('request-access'); - if (reply.ok && reply.status) lastAccess = reply.status; - } catch (e) { - log('computer-host', 'could not request access', { - error: e instanceof Error ? e.message : String(e) - }); + // Linux has no permission prompts: what it needs is the GNOME extension, + // installed and enabled from here. + if (platform === 'linux' && !run) { + try { + await gnome.install(); + } catch (e) { + log('computer-host', 'could not install the GNOME extension', { + error: e instanceof Error ? e.message : String(e) + }); + } } + await checkAccess('request-access'); await announce(); return { ...state(), enabled: await readComputerHostEnabled() }; }, diff --git a/src/desktop/computer-host/store.ts b/src/desktop/computer-host/store.ts index 3365d052..4eedaa5f 100644 --- a/src/desktop/computer-host/store.ts +++ b/src/desktop/computer-host/store.ts @@ -39,6 +39,6 @@ export async function writeComputerHostEnabled(enabled: boolean): Promise await chmod(path, 0o600).catch(() => undefined); log( 'computer-host', - enabled ? 'this Mac now lets Stem drive its screen' : 'this Mac stopped letting Stem drive its screen' + enabled ? 'this computer now lets Stem drive its screen' : 'this computer stopped letting Stem drive its screen' ); } diff --git a/src/desktop/dictation.ts b/src/desktop/dictation/index.ts similarity index 60% rename from src/desktop/dictation.ts rename to src/desktop/dictation/index.ts index 40440db4..c825d8e5 100644 --- a/src/desktop/dictation.ts +++ b/src/desktop/dictation/index.ts @@ -1,14 +1,19 @@ import { BrowserWindow } from 'electron'; -import { log } from '../server/log'; -import { HelperProcess, resolveHelperPath } from './computer-host/helper'; -import type { DictationLanguages, DictationUpdate } from '../shared/types'; +import { log } from '../../server/log'; +import { HelperProcess, resolveHelperPath } from '../computer-host/helper'; +import type { DictationLanguages, DictationUpdate } from '../../shared/types'; +import { cancelWhisper, startWhisper, stopWhisper, whisperAudio, whisperLanguages, type WhisperModelId } from './whisper'; -// Dictation into the composers, on this Mac's microphone through Apple's -// on-device SpeechAnalyzer (native/mac/stem-computer/Dictate.swift). Nothing -// leaves the machine and nothing is bundled: the helper the computer-control -// persona already ships gains dictate-* commands, and a process of its own is -// spawned per dictation so a computer-control run and a dictation never share -// one. Client-owned like the recorder: the microphone is this machine's. +const mac = process.platform === 'darwin'; + +// Dictation into the composers, on this computer's microphone. On a Mac it is +// Apple's on-device SpeechAnalyzer (native/mac/stem-computer/Dictate.swift): +// nothing is bundled, the helper the computer-control persona already ships +// gains dictate-* commands, and a process of its own is spawned per dictation +// so a computer-control run and a dictation never share one. On Windows and +// Linux it is Whisper on the CPU (whisper.ts), downloaded on first use, fed by +// the microphone the preload captures. Nothing leaves the machine either way. +// Client-owned like the recorder: the microphone is this machine's. // // Updates go to every window ('dictation:update'), tagged with the session // they belong to; the composer that started it keeps its own and ignores the @@ -29,7 +34,7 @@ function release(): void { } export async function dictationLanguages(): Promise { - if (process.platform !== 'darwin') return { available: false, reason: 'Dictation is only available on macOS.', languages: [], current: '' }; + if (!mac) return whisperLanguages(); const proc = new HelperProcess(await resolveHelperPath()); try { const reply = await proc.call('dictate-languages'); @@ -41,7 +46,8 @@ export async function dictationLanguages(): Promise { } /** Starts listening; resolves with the session id once the model is ready. Rejects with the reason. */ -export async function startDictation(locale: string | null): Promise<{ session: number; locale: string }> { +export async function startDictation(locale: string | null, model: WhisperModelId = 'fast'): Promise<{ session: number; locale: string }> { + if (!mac) return startWhisper(locale, model, { broadcast }); release(); const id = ++session; const proc = new HelperProcess(await resolveHelperPath()); @@ -62,6 +68,7 @@ export async function startDictation(locale: string | null): Promise<{ session: /** Stops listening and answers with everything heard. */ export async function stopDictation(): Promise { + if (!mac) return stopWhisper(); const proc = helper; if (!proc) return ''; try { @@ -74,5 +81,12 @@ export async function stopDictation(): Promise { } export function cancelDictation(): void { + if (!mac) return cancelWhisper(); release(); } + +/** Windows and Linux: a block of the microphone, 16 kHz mono, from the preload. */ +export function dictationAudio(samples: unknown): void { + if (mac || !(samples instanceof Float32Array) || samples.length > 16_000 * 10) return; + whisperAudio(samples, { broadcast }); +} diff --git a/src/desktop/dictation/whisper-worker.ts b/src/desktop/dictation/whisper-worker.ts new file mode 100644 index 00000000..2474a363 --- /dev/null +++ b/src/desktop/dictation/whisper-worker.ts @@ -0,0 +1,79 @@ +// Utility-process entry for dictation off the Mac: Whisper through +// transformers.js on onnxruntime-node's CPU provider, the same stack the +// recall embedder runs (server/recall/embed-worker.ts). A process of its own +// so a decode never stalls the main process, and so its memory goes away with +// it. Talks to desktop/dictation/whisper.ts over process.parentPort. +// +// This file must stay free of Electron imports beyond the ambient parentPort. + +export interface WhisperModelSpec { + /** HF repo with transformers.js-compatible ONNX weights. */ + repo: string; + encoder: 'q4' | 'q8' | 'fp16' | 'fp32'; + decoder: 'q4' | 'q8' | 'fp16' | 'fp32'; +} + +export type WhisperInMessage = + | { type: 'load'; spec: WhisperModelSpec; cacheDir: string } + | { type: 'transcribe'; id: number; audio: Float32Array; language: string | null }; + +export type WhisperOutMessage = + | { type: 'progress'; downloading: boolean } + | { type: 'ready' } + | { type: 'load-failed'; error: string } + | { type: 'text'; id: number; text: string } + | { type: 'failed'; id: number; error: string }; + +type Asr = (audio: Float32Array, options: Record) => Promise<{ text: string } | { text: string }[]>; + +const port = process.parentPort as NonNullable; +let asr: Asr | null = null; + +const post = (m: WhisperOutMessage): void => port.postMessage(m); + +async function load(spec: WhisperModelSpec, cacheDir: string): Promise { + try { + const { pipeline, env } = await import('@huggingface/transformers'); + env.cacheDir = cacheDir; + let announced = false; + asr = (await pipeline('automatic-speech-recognition', spec.repo, { + dtype: { encoder_model: spec.encoder, decoder_model_merged: spec.decoder }, + // A file being fetched (not read from the cache) means a first use: say so once. + progress_callback: (p: { status?: string }) => { + if (!announced && p.status === 'download') { + announced = true; + post({ type: 'progress', downloading: true }); + } + } + })) as unknown as Asr; + post({ type: 'ready' }); + } catch (e) { + post({ type: 'load-failed', error: e instanceof Error ? e.message : String(e) }); + } +} + +async function transcribe(id: number, audio: Float32Array, language: string | null): Promise { + if (!asr) { + post({ type: 'failed', id, error: 'The speech model is not loaded.' }); + return; + } + try { + const out = await asr(audio, { + task: 'transcribe', + // Null lets Whisper tell the language from the speech. + ...(language ? { language } : {}), + chunk_length_s: 30, + return_timestamps: false + }); + const text = (Array.isArray(out) ? out.map((o) => o.text).join(' ') : out.text).trim(); + post({ type: 'text', id, text }); + } catch (e) { + post({ type: 'failed', id, error: e instanceof Error ? e.message : String(e) }); + } +} + +port.on('message', (event: { data: WhisperInMessage }) => { + const m = event.data; + if (m.type === 'load') void load(m.spec, m.cacheDir); + else if (m.type === 'transcribe') void transcribe(m.id, m.audio, m.language); +}); diff --git a/src/desktop/dictation/whisper.ts b/src/desktop/dictation/whisper.ts new file mode 100644 index 00000000..a972c771 --- /dev/null +++ b/src/desktop/dictation/whisper.ts @@ -0,0 +1,270 @@ +import { app } from 'electron'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { host, type WorkerTransport } from '../../server/host'; +import { log } from '../../server/log'; +import type { DictationLanguages, DictationUpdate } from '../../shared/types'; +import type { WhisperInMessage, WhisperModelSpec, WhisperOutMessage } from './whisper-worker'; + +// Dictation off the Mac: Whisper on this computer's CPU (whisper-worker.ts), +// fed by the microphone the preload captures (16 kHz mono, see +// preload/dictation-capture.ts). Nothing leaves the machine; the model is +// downloaded once, on the first dictation, into this profile's state folder. +// +// Whisper has no streaming mode, so the live text is a re-transcription of +// the stretch being spoken: as soon as one decode finishes, the next starts on +// everything heard since the stretch began, which paces itself to the +// computer's speed. A stretch longer than SEGMENT_S is settled ("final") and +// a new one begins, so a decode never grows past Whisper's 30-second window. + +const SAMPLE_RATE = 16_000; +const SEGMENT_S = 24; +/** Too little new speech to be worth another pass. */ +const MIN_NEW_S = 0.4; +/** The worker (and the model in its memory) outlives a dictation by this much. */ +const IDLE_MS = 5 * 60_000; + +export type WhisperModelId = 'fast' | 'accurate'; + +/** The two models Settings offers: speed against accuracy (and download size). */ +export const WHISPER_MODELS: Record = { + // 240 MB; about a quarter of real time on a 6-core laptop CPU. + fast: { repo: 'onnx-community/whisper-small', encoder: 'q8', decoder: 'q8' }, + // 730 MB; about 80% of real time on the same CPU, much better outside English. + accurate: { repo: 'onnx-community/whisper-large-v3-turbo', encoder: 'q4', decoder: 'q4' } +}; + +/** Languages offered in Settings: Whisper's best-supported, by ISO 639-1 code. */ +const LANGUAGES = [ + 'en', 'sk', 'cs', 'de', 'fr', 'es', 'it', 'pt', 'nl', 'pl', 'hu', 'ro', 'bg', 'hr', 'sl', 'sr', 'uk', 'ru', 'el', + 'tr', 'sv', 'da', 'no', 'fi', 'et', 'lv', 'lt', 'ca', 'ar', 'he', 'hi', 'ja', 'ko', 'zh', 'vi', 'th', 'id', 'ms' +]; + +interface Session { + id: number; + language: string | null; + /** Everything heard, in arrival order. */ + chunks: Float32Array[]; + samples: number; + /** Where the stretch being re-transcribed begins (samples). */ + segmentStart: number; + /** The settled text of earlier stretches. */ + finals: string[]; + /** The latest reading of the current stretch. */ + volatile: string; + /** How far the last decode of this stretch reached (samples). */ + decodedTo: number; + decoding: Promise | null; + stopped: boolean; +} + +interface Deps { + broadcast(update: DictationUpdate): void; +} + +let worker: WorkerTransport | null = null; +let loaded: WhisperModelId | null = null; +let ready: Promise | null = null; +let idle: NodeJS.Timeout | null = null; +let nextRequest = 1; +const pending = new Map(); +let onProgress: ((downloading: boolean) => void) | null = null; +let session: Session | null = null; +let sessionCount = 0; + +function cacheDir(): string { + return join(host().stateRoot(), 'speech-models'); +} + +function stopWorker(): void { + worker?.kill(); + worker = null; + loaded = null; + ready = null; + for (const [, p] of pending) p.reject(new Error('The speech model stopped.')); + pending.clear(); +} + +function keepAlive(): void { + if (idle) clearTimeout(idle); + idle = setTimeout(() => { + if (!session) stopWorker(); + }, IDLE_MS); + idle.unref?.(); +} + +/** A worker with this model loaded, starting one (and the download) when needed. */ +function ensureModel(model: WhisperModelId): Promise { + if (worker && loaded === model && ready) return ready; + stopWorker(); + const entry = join(dirname(fileURLToPath(import.meta.url)), 'dictation-worker.js'); + const w = host().forkWorker(entry, { serviceName: 'stem-dictation-worker' }); + worker = w; + loaded = model; + ready = new Promise((resolve, reject) => { + w.onMessage((raw) => { + const m = raw as WhisperOutMessage; + if (m.type === 'progress') onProgress?.(m.downloading); + else if (m.type === 'ready') resolve(); + else if (m.type === 'load-failed') { + log('dictation', 'the speech model did not load', { error: m.error }); + reject(new Error(`The speech model could not be loaded: ${m.error}`)); + } else if (m.type === 'text' || m.type === 'failed') { + const p = pending.get(m.id); + pending.delete(m.id); + if (m.type === 'text') p?.resolve(m.text); + else p?.reject(new Error(m.error)); + } + }); + w.onExit((code) => { + if (worker !== w) return; + log('dictation', 'the speech worker exited', { code }); + reject(new Error('The speech model stopped.')); + stopWorker(); + }); + }); + const load: WhisperInMessage = { type: 'load', spec: WHISPER_MODELS[model], cacheDir: cacheDir() }; + w.send(load); + // A failed load must not stay cached as the answer for the next dictation. + ready.catch(() => { + if (worker === w) stopWorker(); + }); + return ready; +} + +function transcribe(audio: Float32Array, language: string | null): Promise { + const w = worker; + if (!w) return Promise.reject(new Error('The speech model is not loaded.')); + const id = nextRequest++; + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }); + const m: WhisperInMessage = { type: 'transcribe', id, audio, language }; + w.send(m); + }); +} + +/** The samples [from, to) of the session's audio, as one array. */ +function slice(s: Session, from: number, to: number): Float32Array { + const out = new Float32Array(Math.max(0, to - from)); + let at = 0; + let filled = 0; + for (const c of s.chunks) { + const start = Math.max(from, at); + const end = Math.min(to, at + c.length); + if (end > start) { + out.set(c.subarray(start - at, end - at), filled); + filled += end - start; + } + at += c.length; + if (at >= to) break; + } + return out; +} + +const heard = (s: Session): string => [...s.finals, s.volatile].filter(Boolean).join(' '); + +/** One more pass over the current stretch, then another while speech keeps coming. */ +function pump(s: Session, deps: Deps): void { + if (s.decoding || s.stopped || loaded === null || !worker) return; + if (s.samples - s.decodedTo < MIN_NEW_S * SAMPLE_RATE) return; + const to = s.samples; + const settle = to - s.segmentStart >= SEGMENT_S * SAMPLE_RATE; + s.decoding = transcribe(slice(s, s.segmentStart, to), s.language) + .then((text) => { + if (session !== s) return; + if (settle) { + if (text) s.finals.push(text); + s.volatile = ''; + s.segmentStart = to; + } else { + s.volatile = text; + } + s.decodedTo = to; + deps.broadcast({ session: s.id, final: s.finals.join(' '), volatile: s.volatile ? ` ${s.volatile}` : '' }); + }) + .catch((e: unknown) => log('dictation', 'a live pass failed', { error: e instanceof Error ? e.message : String(e) })) + .finally(() => { + s.decoding = null; + pump(s, deps); + }); +} + +/** Whisper's code for a locale, or null to let it tell the language from the speech. */ +function languageOf(locale: string | null): string | null { + const code = (locale || app.getLocale()).split(/[-_]/)[0]?.toLowerCase() ?? ''; + return LANGUAGES.includes(code) ? code : null; +} + +export function whisperLanguages(): DictationLanguages { + const names = new Intl.DisplayNames([app.getLocale() || 'en'], { type: 'language' }); + const languages = LANGUAGES.map((id) => ({ id, name: names.of(id) ?? id })).sort((a, b) => a.name.localeCompare(b.name)); + const current = (app.getLocale().split('-')[0] ?? 'en').toLowerCase(); + return { available: true, languages, current: LANGUAGES.includes(current) ? current : 'en' }; +} + +/** Listening starts at once (the preload is already capturing); resolves when the model is ready. */ +export async function startWhisper(locale: string | null, model: WhisperModelId, deps: Deps): Promise<{ session: number; locale: string }> { + cancelWhisper(); + const s: Session = { + id: ++sessionCount, + language: languageOf(locale), + chunks: [], + samples: 0, + segmentStart: 0, + finals: [], + volatile: '', + decodedTo: 0, + decoding: null, + stopped: false + }; + session = s; + onProgress = (downloading) => { + if (downloading && session === s) deps.broadcast({ session: s.id, final: '', volatile: '', downloading: true }); + }; + try { + await ensureModel(model); + } catch (e) { + if (session === s) session = null; + throw e; + } + if (session !== s) throw new Error('Dictation was stopped before it started.'); + pump(s, deps); + return { session: s.id, locale: s.language ?? '' }; +} + +/** A block of 16 kHz mono samples from the preload's capture. */ +export function whisperAudio(samples: Float32Array, deps: Deps): void { + const s = session; + if (!s || s.stopped) return; + s.chunks.push(samples); + s.samples += samples.length; + if (ready && loaded) pump(s, deps); +} + +/** Everything heard: the live pass finishes, and one last pass reads the rest of the stretch. */ +export async function stopWhisper(): Promise { + const s = session; + if (!s) return ''; + s.stopped = true; + try { + await s.decoding; + if (worker && s.samples > s.segmentStart && s.samples > s.decodedTo) { + s.volatile = await transcribe(slice(s, s.segmentStart, s.samples), s.language); + } + return heard(s); + } finally { + if (session === s) session = null; + keepAlive(); + } +} + +export function cancelWhisper(): void { + if (session) session.stopped = true; + session = null; + keepAlive(); +} + +export function closeWhisper(): void { + cancelWhisper(); + stopWorker(); +} diff --git a/src/desktop/exec-host/index.ts b/src/desktop/exec-host/index.ts index d819f5c0..100f7dd8 100644 --- a/src/desktop/exec-host/index.ts +++ b/src/desktop/exec-host/index.ts @@ -1,7 +1,7 @@ import { mkdir, readdir, rm, stat } from 'node:fs/promises'; import { join } from 'node:path'; import { log } from '../../server/log'; -import { clampTimeout, execEnv, resolveLoginPath, runCommand } from '../../server/exec/executor'; +import { clampTimeout, execEnv, resolveLoginPath, runCommand, timedOutLine } from '../../server/exec/executor'; import { scanCommandAgainstRoots } from '../../server/exec/protected'; import { hostShellFromPlatform } from '../../server/exec/host-shell'; import { host } from '../../server/host'; @@ -159,7 +159,7 @@ export function createExecHost(deps: ExecHostDeps): ExecHost { // model reads one format wherever a command ran. const parts = [ outcome.timedOut - ? `Timed out after ${timeoutMs} ms (process group killed).` + ? timedOutLine(timeoutMs) : `Exit code: ${outcome.exitCode ?? `signal ${outcome.signal ?? 'unknown'}`}`, `stdout:\n${outcome.stdout.trim() || '(no output)'}`, `stderr:\n${outcome.stderr.trim() || '(no output)'}` diff --git a/src/desktop/host.ts b/src/desktop/host.ts index 1506e341..faff2478 100644 --- a/src/desktop/host.ts +++ b/src/desktop/host.ts @@ -1,5 +1,6 @@ import { app, safeStorage, shell, utilityProcess } from 'electron'; import type { StemHost, WorkerTransport } from '../server/host'; +import { log } from '../server/log'; // The Electron half of the host shim (src/server/host/index.ts). Everything the // server used to reach for directly, supplied by the process that actually has @@ -58,7 +59,11 @@ export function electronHost(): Partial { // Electron's own child windows. nodeSpawn: () => ({ command: process.execPath, env: { ELECTRON_RUN_AS_NODE: '1' } }), openExternal: (url) => { - void shell.openExternal(url); + shell.openExternal(url).catch((e: unknown) => { + // A sign-in that opens no browser leaves the person at "paste the code" + // with nothing to paste: say why in the log at least. + log('desktop', 'could not open the browser', { error: e instanceof Error ? e.message : String(e) }); + }); }, onShutdown: (fn) => { app.on('will-quit', () => fn()); diff --git a/src/desktop/index.ts b/src/desktop/index.ts index f3e1ad22..4123f501 100644 --- a/src/desktop/index.ts +++ b/src/desktop/index.ts @@ -2,6 +2,7 @@ import { app, BrowserWindow, clipboard, + dialog, globalShortcut, ipcMain, nativeImage, @@ -12,7 +13,7 @@ import { } from 'electron'; import { join } from 'node:path'; import { mkdirSync } from 'node:fs'; -import { installAppMenu } from './app-menu'; +import { IN_WINDOW_SHORTCUTS, installAppMenu, updateAppMenu } from './app-menu'; import { installContextMenu } from './context-menu'; import { openLink } from './open-link'; import { electronHost } from './host'; @@ -20,7 +21,7 @@ import { setHost } from '../server/host'; import { startServer, type ServerHandle } from '../server'; import { log } from '../server/log'; import { resolveProfileOverride } from '../server/workspace/paths'; -import { bindServerChannels } from './ipc-bridge'; +import { bindServerChannels, handleLocal } from './ipc-bridge'; import { registerLocalIpc } from './local'; import { createMcpHost, type McpHost } from './mcp-host'; import { createExecHost, type ExecHost } from './exec-host'; @@ -44,7 +45,7 @@ import { createQuickChat } from './quickchat'; import { loadRenderer, PRELOAD_SCRIPT } from './renderer-assets'; import { initTray } from './tray'; import { RendererPushQueue } from './ui-lifecycle'; -import type { AppSettings, ThemeState } from '../shared/types'; +import type { AppSettings, MenuCommand, ThemeState } from '../shared/types'; // The Electron main process: windows, tray, the global shortcut, and app // lifecycle. It owns nothing about chats, memory, skills or settings — it starts @@ -65,6 +66,9 @@ import type { AppSettings, ThemeState } from '../shared/types'; // "Electron" — that comes from the Electron.app bundle and only changes when // the app is packaged. app.setName('Stem'); +// Windows ties notifications and the taskbar to the Start-menu shortcut by its +// AppUserModelID, which the installer sets to the appId (electron-builder.yml). +if (process.platform === 'win32') app.setAppUserModelId('com.join3r.stem'); // Background (E2E) mode: keep a test run out of the way of whoever is using the // machine. On macOS a launching regular app makes itself frontmost, and when it @@ -108,7 +112,8 @@ if (profileOverride) { // One Stem per profile. A second launch hands its argv to the running instance // and exits: `stem --quick-chat` toggles the overlay (the summon path for // Wayland, where Electron's globalShortcut never fires — users bind a DE -// shortcut to this command), a plain `stem` reveals the main window (the +// shortcut to this command), `stem --ask-window` is "Ask about this window" +// the same way, a plain `stem` reveals the main window (the // reopen path on Linux, where the app keeps running after the main window // closes and stock GNOME hides the tray). Must run after the profile override // so the lock keys off the final userData path (isolated E2E/profile runs @@ -119,6 +124,7 @@ app.on('second-instance', (_event, argv) => { // Ignore launches racing the whenReady bootstrap (windows not created yet). if (!windowsReady) return; if (argv.includes('--quick-chat')) quickChat.toggle(); + else if (argv.includes('--ask-window')) quickChat.askAboutWindow(); else revealMainWindow(); }); @@ -297,6 +303,12 @@ function sendToMain(channel: string, payload: unknown): void { } } +/** The summon key to print beside Quick Chat in the menu — only one the OS granted. */ +function liveQuickChatAccelerator(): string | null { + const status = quickChat.shortcutStatus(); + return status.registered ? status.accelerator : null; +} + /** Bring the main window to the front (recreating it if it was closed). */ function revealMainWindow(): void { if (!mainWindow || mainWindow.isDestroyed()) createWindow(); @@ -321,6 +333,55 @@ const updates = createUpdates({ openExternal: openExternalUrl }); +/** + * Run a menu pick in the main window. Commands about what is on screen (the + * composer, the selection) only make sense while the main window is the one in + * front; picked over the Quick Chat overlay they are dropped rather than reaching + * into a window the user is not looking at. Everything else brings the main + * window forward first, reopening it if it was closed. + */ +function runMenuCommand(command: MenuCommand): void { + if (command.kind === 'shortcut' && IN_WINDOW_SHORTCUTS.has(command.id)) { + const focused = BrowserWindow.getFocusedWindow(); + if (!focused || focused !== mainWindow) return; + } else { + revealMainWindow(); + } + sendToMain('menu:command', command); +} + +/** + * Check for Updates… from the menu. Unlike the background check, someone asked, + * so every outcome gets an answer — including "nothing newer", which the + * renderer's update prompt never shows. + */ +async function checkForUpdatesFromMenu(): Promise { + const status = await updates.check(); + const parent = mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined; + const show = (options: Electron.MessageBoxOptions) => + parent ? dialog.showMessageBox(parent, options) : dialog.showMessageBox(options); + if (status.state === 'error') { + await show({ type: 'warning', message: 'Couldn’t check for updates', detail: status.error ?? undefined }); + return; + } + if (!status.available) { + await show({ type: 'info', message: 'Stem is up to date', detail: `Version ${status.appVersion} is the newest.` }); + return; + } + const ready = status.state === 'ready' || status.mode === 'manual'; + const { response } = await show({ + type: 'info', + message: `Stem ${status.available} is available`, + detail: ready + ? `You have ${status.appVersion}.` + : `You have ${status.appVersion}. It is downloading; Stem will offer to restart when it is ready.`, + buttons: ready ? [status.mode === 'manual' ? 'Download' : 'Restart Now', 'Later'] : ['OK'], + defaultId: 0, + cancelId: ready ? 1 : 0 + }); + if (ready && response === 0) await updates.install(); +} + const quickChat = createQuickChat({ mainWindow: () => mainWindow, sendToMain, @@ -391,10 +452,17 @@ app.whenReady().then(async () => { app.dock?.setIcon(appIcon); } - // Stem's own menu, which is the stock one minus Close Window — no ⌘W closing + // Stem's own menu: every shortcut listed, and no Close Window — no ⌘W closing // the app's only window (or the Quick Chat overlay, which never comes back). // See app-menu.ts. - installAppMenu(); + installAppMenu( + { + command: runMenuCommand, + toggleQuickChat: () => quickChat.toggle(), + checkForUpdates: () => void checkForUpdatesFromMenu() + }, + { canCheckForUpdates: updates.status().mode !== 'none' } + ); // Take this machine's own settings out of settings.json before anything can // rewrite that file without them. The migration runs exactly once and this is @@ -518,7 +586,10 @@ app.whenReady().then(async () => { hostScript: BROWSER_NATIVE_HOST_SCRIPT, nodeCommand: process.execPath }), - copyText: (text) => clipboard.writeText(text) + copyText: (text) => clipboard.writeText(text), + // Tests press Set up for real; it must not open the file manager or start + // a browser on the machine running them. + ...(E2E ? { openWith: async () => undefined } : {}) }); proxy = createServerProxy({ @@ -545,7 +616,10 @@ app.whenReady().then(async () => { revealMainWindow, requestAttention: () => requestAttention(mainWindow), threadOpened: (threadId) => quickChat.threadOpened(threadId), - applyQuickChatSettings: (patch, next) => quickChat.applySettings(patch, next), + applyQuickChatSettings: (patch, next) => { + quickChat.applySettings(patch, next); + updateAppMenu({ quickChatAccelerator: liveQuickChatAccelerator() }); + }, // Renderer recovery keeps its existing replay order. The HUD uses the // separate post-replay snapshot above, with authenticated device identity. resync: () => sendToMain('client:resync', undefined), @@ -620,6 +694,10 @@ app.whenReady().then(async () => { if (themeState.custom) themeChanged(themeState); }); quickChat.registerIpc(); + // The rows ⌘1–9 open, pushed by the chat list whenever they change. + handleLocal('menu:setChats', (_e, threadIds: string[], titles: string[]) => { + updateAppMenu({ chats: threadIds.slice(0, 9).map((threadId, i) => ({ threadId, title: titles[i] ?? '' })) }); + }); ipcMain.on('renderer:ready', (event) => { const win = mainWindow; if (!win || win.isDestroyed() || event.sender !== win.webContents) return; @@ -639,7 +717,8 @@ app.whenReady().then(async () => { // that command IS the shortcut on Wayland (see the second-instance handler), // and pressing it with Stem closed should feel the same as pressing it with // Stem running. The overlay is summoned once it exists, below. - const coldSummon = process.argv.includes('--quick-chat'); + // `--ask-window` cold: there is no helper warm yet, so it is a plain summon. + const coldSummon = process.argv.includes('--quick-chat') || process.argv.includes('--ask-window'); createWindow(coldSummon); // Eagerly spawn pi + connect MCP once the window has painted, so the first prompt // doesn't pay backend cold-start. did-finish-load keeps the spawn + MCP child @@ -664,6 +743,7 @@ app.whenReady().then(async () => { }); quickChat.start(settings.quickChat); + updateAppMenu({ quickChatAccelerator: liveQuickChatAccelerator() }); // After the windows exist and the first prompt's warm-up is on its way; the // first check waits a further beat inside (FIRST_CHECK_DELAY_MS). updates.start(); @@ -725,6 +805,8 @@ app.on('before-quit', (event) => { browserHost?.close(); // And a recording in progress: its helper is a child too; its folder stays for recover(). recorder?.close(); + // And the "Ask about this window" helper. + quickChat.stop(); // Nothing to drain when the server is somebody else's process. if (!server) return; event.preventDefault(); diff --git a/src/desktop/ipc-bridge.ts b/src/desktop/ipc-bridge.ts index 77e0bc9d..159edae0 100644 --- a/src/desktop/ipc-bridge.ts +++ b/src/desktop/ipc-bridge.ts @@ -64,6 +64,8 @@ const LOCAL_IPC_ARGS: Record = { // The browser host's switch and browser pick ('browserHost:localState' / ':setUp' take no arguments). 'browserHost:setEnabled': [a.boolean], 'browserHost:choose': [a.string], + 'browserHost:rename': [a.string, a.string], + 'browserHost:forget': [a.string], 'browserHost:openExtensions': [a.optional(a.nullish(a.string))], // Folders this machine mirrors: the native picker's absolute paths. // ('mirror:localState' takes no arguments, so it is absent.) @@ -74,11 +76,19 @@ const LOCAL_IPC_ARGS: Record = { // ':pause' / ':cancel' take no arguments): the chat it records for, and the // draft a second example is added to. 'recorder:start': [a.string, a.optional(a.nullish(a.string))], - // Dictation on this Mac ('dictation:languages' / ':stop' / ':cancel' take no - // arguments): the language to listen in, null for the Mac's own. - 'dictation:start': [a.nullish(a.string)], + // Dictation ('dictation:languages' / ':stop' / ':cancel' take no + // arguments): the language to listen in, null for the computer's own, and + // off a Mac which Whisper model; then the microphone, 16 kHz mono blocks + // (a Float32Array, checked again in the handler). + 'dictation:start': [a.nullish(a.string), a.optional(a.oneOf(['fast', 'accurate']))], + 'dictation:audio': [a.object], 'quickchat:run': [a.object], - 'quickchat:handoff': [a.object] + 'quickchat:handoff': [a.object], + // Whether the compact bar keeps room for the window chip ("Ask about this window"). + 'quickchat:compact': [a.optional(a.boolean)], + // The System Settings pane the chip's missing-permission link opens. + 'quickchat:openPrivacy': [a.oneOf(['screen', 'accessibility'])], + 'menu:setChats': [a.stringArray, a.stringArray] }; /** @@ -181,5 +191,9 @@ export function bindServerChannels( */ const NEEDS_A_NEWER_SERVER: Record = { 'mcp:setLocation': - 'This Stem server is older than this copy of Stem and cannot pin an MCP server to a computer. Update the server (or its container) and try again.' + 'This Stem server is older than this copy of Stem and cannot pin an MCP server to a computer. Update the server (or its container) and try again.', + 'mcp:setApps': + 'This Stem server is older than this copy of Stem and cannot match MCP servers to apps. Update the server (or its container) and try again.', + 'mcp:forApp': + 'This Stem server is older than this copy of Stem and cannot match MCP servers to apps. Update the server (or its container) and try again.' }; diff --git a/src/desktop/local/index.ts b/src/desktop/local/index.ts index 15186650..ff62d535 100644 --- a/src/desktop/local/index.ts +++ b/src/desktop/local/index.ts @@ -10,7 +10,7 @@ import { sampleFolder } from '../../server/connected-folders/sample'; import { workspaceRoot } from '../../server/workspace/paths'; import { exportState } from '../../server/workspace/state-transfer'; import { readClientIdentity, storedServerUrl } from '../client-store'; -import { cancelDictation, dictationLanguages, startDictation, stopDictation } from '../dictation'; +import { cancelDictation, dictationAudio, dictationLanguages, startDictation, stopDictation } from '../dictation'; import { downloadFile } from '../file-transfer'; import { markReleaseNotesRead, releaseNotesSnapshot } from '../release-notes'; import { downloadsDir, saveImageToDownloads } from '../save-image'; @@ -271,7 +271,8 @@ export function registerLocalIpc(deps: LocalIpcDeps): void { // Dictation: client-owned because the microphone is this machine's; the // server only ever sees the text, once it is sent. handleLocal('dictation:languages', () => dictationLanguages()); - handleLocal('dictation:start', (_e, locale: string | null) => startDictation(locale ?? null)); + handleLocal('dictation:start', (_e, locale: string | null, model?: 'fast' | 'accurate') => startDictation(locale ?? null, model ?? 'fast')); + handleLocal('dictation:audio', (_e, samples: unknown) => dictationAudio(samples)); handleLocal('dictation:stop', () => stopDictation()); handleLocal('dictation:cancel', () => cancelDictation()); handleLocal('computerHost:requestAccess', (): Promise => deps.computerHost.requestAccess()); @@ -285,6 +286,11 @@ export function registerLocalIpc(deps: LocalIpcDeps): void { (_e, enabled: boolean): Promise => deps.browserHost.setEnabled(enabled) ); handleLocal('browserHost:choose', (_e, id: string): Promise => deps.browserHost.choose(id)); + handleLocal( + 'browserHost:rename', + (_e, id: string, name: string): Promise => deps.browserHost.rename(id, name) + ); + handleLocal('browserHost:forget', (_e, id: string): Promise => deps.browserHost.forget(id)); handleLocal('browserHost:setUp', (): Promise => deps.browserHost.setUp()); handleLocal('browserHost:openExtensions', (_e, id?: string | null): Promise => deps.browserHost.openExtensionsPage(id ?? undefined) diff --git a/src/desktop/mirror-host/index.ts b/src/desktop/mirror-host/index.ts index 3f00a4de..ecfd9a9b 100644 --- a/src/desktop/mirror-host/index.ts +++ b/src/desktop/mirror-host/index.ts @@ -64,6 +64,8 @@ interface Entry { folder: MirroredFolder; watcher: FSWatcher | null; timer: NodeJS.Timeout | null; + /** When the pending timer fires (Date.now() ms). */ + dueAt: number; running: boolean; dirty: boolean; frozen: boolean; @@ -80,7 +82,7 @@ export function createMirrorHost(deps: MirrorHostDeps): MirrorHost { function ensureEntry(folder: MirroredFolder): Entry { let entry = entries.get(folder.folderId); if (!entry) { - entry = { folder, watcher: null, timer: null, running: false, dirty: false, frozen: false, lastCount: 0 }; + entry = { folder, watcher: null, timer: null, dueAt: 0, running: false, dirty: false, frozen: false, lastCount: 0 }; entries.set(folder.folderId, entry); } else { entry.folder = folder; @@ -121,7 +123,13 @@ export function createMirrorHost(deps: MirrorHostDeps): MirrorHost { function scheduleSync(folderId: string, delayMs: number): void { const entry = entries.get(folderId); if (!entry || closed) return; + // A sync already due sooner stays put: a steady stream of events must not + // push it back forever. (Windows' watcher fires nonstop once its root is + // deleted, which kept the round that notices the missing root from running.) + const dueAt = Date.now() + delayMs; + if (entry.timer && entry.dueAt <= dueAt) return; if (entry.timer) clearTimeout(entry.timer); + entry.dueAt = dueAt; entry.timer = setTimeout(() => { entry.timer = null; void syncNow(entry); @@ -175,6 +183,9 @@ export function createMirrorHost(deps: MirrorHostDeps): MirrorHost { // last good state, and sync resumes when a scan sees files again. if (scan.rootMissing || (entry.lastCount > 0 && scan.entries.length === 0)) { entry.frozen = true; + // Nothing left to watch; the next successful scan re-arms it. + entry.watcher?.close(); + entry.watcher = null; await deps.invoke('mirror:report', [folderId, { state: 'root-missing' }]); return; } diff --git a/src/desktop/mirror-host/scan.ts b/src/desktop/mirror-host/scan.ts index 575ec8f2..279f1a52 100644 --- a/src/desktop/mirror-host/scan.ts +++ b/src/desktop/mirror-host/scan.ts @@ -1,13 +1,18 @@ -import { readdir, stat } from 'node:fs/promises'; +import { readFile, readdir, stat } from 'node:fs/promises'; import { join } from 'node:path'; +import ignore, { type Ignore } from 'ignore'; import type { MirrorManifestEntry } from '../../shared/types'; // One scan of a mirrored folder on THIS machine: the full tree as // '/'-separated rels with {size, mtimeMs}, minus what deliberately never -// mirrors. The rules are fixed (decision: no .gitignore/.stemignore in v1) -// and every exclusion the user might miss is REPORTED, not silent — the -// skipped list reaches the Folders tab, and the prompt discloses the rules so -// the assistant does not read a missing node_modules as a broken folder. +// mirrors: the fixed names below, and whatever the folder's own .gitignore +// files exclude (build output, caches, local state). A repo's ios/build or +// dist can be thousands of files nobody reads; the project already says so. +// Only .gitignore files inside the folder count — not .git/info/exclude or +// a global excludes file, which belong to one checkout, not the project. +// Every exclusion the user might miss is REPORTED, not silent — the skipped +// list reaches the Folders tab, and the prompt discloses the rules so the +// assistant does not read a missing node_modules as a broken folder. /** Names that never mirror, wherever they appear in the tree. */ export const MIRROR_IGNORED_NAMES = new Set(['.git', 'node_modules', '.DS_Store', 'Thumbs.db', 'desktop.ini']); @@ -35,7 +40,7 @@ export async function scanMirrorRoot(root: string): Promise { if (skipped.length < MAX_SKIPPED) skipped.push({ rel, reason }); }; - async function walk(dir: string, relBase: string): Promise { + async function walk(dir: string, relBase: string, rules: GitignoreRules[]): Promise { let names; try { names = await readdir(dir, { withFileTypes: true }); @@ -43,9 +48,15 @@ export async function scanMirrorRoot(root: string): Promise { skip(relBase || '.', 'unreadable directory'); return; } + if (names.some((d) => d.name === '.gitignore' && d.isFile())) { + const text = await readFile(join(dir, '.gitignore'), 'utf8').catch(() => null); + if (text !== null) rules = [...rules, { base: relBase, ig: ignore().add(text) }]; + } for (const d of names) { if (MIRROR_IGNORED_NAMES.has(d.name)) continue; // the fixed rules; disclosed, not reported const rel = relBase ? `${relBase}/${d.name}` : d.name; + // Not entering an ignored directory matches git: nothing under it can be re-included. + if (!d.isSymbolicLink() && gitignored(rules, rel, d.isDirectory())) continue; // A symlink is skipped whatever it points at: following one could walk // out of the folder the user picked, which is the one thing a mirror // must never upload. @@ -54,7 +65,7 @@ export async function scanMirrorRoot(root: string): Promise { continue; } if (d.isDirectory()) { - await walk(join(dir, d.name), rel); + await walk(join(dir, d.name), rel, rules); continue; } if (!d.isFile()) { @@ -74,6 +85,24 @@ export async function scanMirrorRoot(root: string): Promise { } } - await walk(root, ''); + await walk(root, '', []); return { entries, skipped, rootMissing: false }; } + +/** One .gitignore's patterns, which apply to paths relative to the directory holding it. */ +interface GitignoreRules { + base: string; + ig: Ignore; +} + +/** Whether git would ignore `rel`: the deepest .gitignore with an opinion wins, so a `!pattern` can re-include. */ +export function gitignored(rules: GitignoreRules[], rel: string, isDir: boolean): boolean { + let ignored = false; + for (const { base, ig } of rules) { + const path = base ? rel.slice(base.length + 1) : rel; + const verdict = ig.test(isDir ? `${path}/` : path); + if (verdict.ignored) ignored = true; + else if (verdict.unignored) ignored = false; + } + return ignored; +} diff --git a/src/desktop/offline-cache.ts b/src/desktop/offline-cache.ts index 697d71d4..ec2f7630 100644 --- a/src/desktop/offline-cache.ts +++ b/src/desktop/offline-cache.ts @@ -226,12 +226,13 @@ export function createOfflineCache({ enabled }: { enabled: boolean }): OfflineCa function open(): DatabaseSync | null { if (db !== undefined) return db; db = null; + let handle: DatabaseSync | null = null; try { const path = chatCachePath(); // The state root exists by the time anything gets here, but a // STEM_CHAT_CACHE_FILE pointed at a fresh directory need not. mkdirSync(dirname(path), { recursive: true }); - const handle = new DatabaseSync(path); + handle = new DatabaseSync(path); handle.exec('PRAGMA journal_mode = WAL;'); const version = (handle.prepare('PRAGMA user_version').get() as { user_version?: number } | undefined) ?.user_version; @@ -257,7 +258,16 @@ export function createOfflineCache({ enabled }: { enabled: boolean }): OfflineCa seedWatermarks(); } catch (e) { // A corrupt file, a read-only disk, a directory that isn't there yet. - // "No cache" is a supported state; failing to start is not. + // "No cache" is a supported state; failing to start is not. The handle + // that opened but could not be used goes too: on Windows it would keep + // the broken file locked for as long as the app runs. + if (db === null) { + try { + handle?.close(); + } catch { + // quiet: a handle that will not close was never usable. + } + } log('cache', 'the offline chat cache could not be opened', { error: String((e as Error)?.message ?? e) }); diff --git a/src/desktop/open-link.ts b/src/desktop/open-link.ts index 250fcbd9..c497efc8 100644 --- a/src/desktop/open-link.ts +++ b/src/desktop/open-link.ts @@ -37,6 +37,9 @@ export function classifyLink(url: string): LinkAction { } if (EXTERNAL_URL_PROTOCOLS.has(parsed.protocol)) return { kind: 'external', url: parsed.toString() }; if (parsed.protocol !== 'file:') return { kind: 'ignore' }; + // A host names another machine. POSIX refuses one in fileURLToPath; Windows + // turns it into a UNC path and would open a file off someone's SMB share. + if (parsed.hostname) return { kind: 'ignore' }; let path: string; try { path = fileURLToPath(parsed); diff --git a/src/desktop/platform.ts b/src/desktop/platform.ts index 309204b5..fc500dc3 100644 --- a/src/desktop/platform.ts +++ b/src/desktop/platform.ts @@ -58,13 +58,13 @@ export function enableGlobalShortcutPortal(): void { * one itself (Wayland). A second launch hands its argv to the running instance, * which toggles the overlay — see the second-instance handler in index.ts. */ -export function quickChatSummonCommand(): string { +export function quickChatSummonCommand(flag: '--quick-chat' | '--ask-window' = '--quick-chat'): string { // Inside an AppImage, execPath points at the extracted temp mount, which is // gone by the time the user's keybinding fires. $APPIMAGE is the stable file. const exe = process.env.APPIMAGE || process.execPath; const quoted = /\s/.test(exe) ? `"${exe}"` : exe; // Unpackaged (dev) runs need the app directory too, or Electron opens its demo. - return app.isPackaged ? `${quoted} --quick-chat` : `${quoted} ${app.getAppPath()} --quick-chat`; + return app.isPackaged ? `${quoted} ${flag}` : `${quoted} ${app.getAppPath()} ${flag}`; } /** diff --git a/src/desktop/proxy.ts b/src/desktop/proxy.ts index 0c56975d..21226136 100644 --- a/src/desktop/proxy.ts +++ b/src/desktop/proxy.ts @@ -381,6 +381,37 @@ function jpegName(name: string): string { return name.replace(/\.(heic|heif|hif)$/i, '') + '.jpg'; } +/** + * POST a JSON body and read the whole answer, over node:http(s) rather than + * fetch. Node's fetch drops any response whose headers take longer than five + * minutes (undici's headersTimeout), whatever signal the caller passes, and an + * RPC may legitimately wait that long: a sign-in waits on the person in the + * browser. A slow ChatGPT login was cut off that way, and onboarding sat on + * "Setting things up…" while the server finished the sign-in nobody was + * waiting for any more. node:http has no such default; `signal` is the bound. + */ +function postJson( + url: string, + headers: Record, + body: string, + signal: AbortSignal +): Promise<{ status: number; text: string }> { + return new Promise((resolve, reject) => { + const req = openStream( + url, + { method: 'POST', headers: { ...headers, 'content-length': String(Buffer.byteLength(body)) }, signal }, + (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode ?? 0, text: Buffer.concat(chunks).toString('utf8') })); + res.on('error', reject); + } + ); + req.on('error', reject); + req.end(body); + }); +} + function openStream(url: string, options: Parameters[1], onRes: Parameters[2]) { return (url.startsWith('https:') ? httpsRequest : httpRequest)(url, options, onRes); } @@ -530,18 +561,16 @@ export function createServerProxy(deps: ProxyDeps): ServerProxy { // ---- POST /rpc ---- async function post(channel: string, args: unknown[], signal?: AbortSignal): Promise { - let res: Response; + let res: { status: number; text: string }; try { - res = await fetch(`${base}/rpc`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: auth }, - body: JSON.stringify({ channel, args }), + res = await postJson( + `${base}/rpc`, + { 'content-type': 'application/json', authorization: auth }, + JSON.stringify({ channel, args }), // A caller's own signal (a cancellable prefetch) on top of the timeout // every call gets. Either one aborting aborts the request. - signal: signal - ? AbortSignal.any([signal, AbortSignal.timeout(RPC_TIMEOUT_MS)]) - : AbortSignal.timeout(RPC_TIMEOUT_MS) - }); + signal ? AbortSignal.any([signal, AbortSignal.timeout(RPC_TIMEOUT_MS)]) : AbortSignal.timeout(RPC_TIMEOUT_MS) + ); } catch (e) { // Nothing on the other end answered — as opposed to answering badly. This // is the ONLY place the cache is allowed to speak, and the reason is the @@ -555,13 +584,18 @@ export function createServerProxy(deps: ProxyDeps): ServerProxy { // the renderer could not have seen before the split. throw new Error(`Stem's server is unreachable: ${String((e as Error)?.message ?? e)}`); } - const body = (await res.json().catch(() => null)) as - | { ok?: boolean; result?: unknown; error?: string } - | null; + type RpcReply = { ok?: boolean; result?: unknown; error?: string } | null; + let body: RpcReply; + try { + body = JSON.parse(res.text) as RpcReply; + } catch { + // Not JSON: the status line below says what happened. + body = null; + } // Anything with a status line came from the server, including a 401 and a // 500. It is up. setReachable(true); - if (res.ok && body?.ok) { + if (res.status >= 200 && res.status < 300 && body?.ok) { // The server answered, so it is up. That does not make the stream healthy — // a dead stream means missed events — so re-open it now rather than waiting // out the backoff. @@ -862,8 +896,20 @@ export function createServerProxy(deps: ProxyDeps): ServerProxy { if (typeof frame.threadId !== 'string' || !frame.threadId) return null; const action = frame.action as { kind?: unknown } | undefined; if (!action || typeof action !== 'object' || typeof action.kind !== 'string') return null; - // As with the screen: the extension refuses what it cannot do. - return { requestId: frame.requestId, threadId: frame.threadId, action: frame.action! }; + // As with the screen: the extension refuses what it cannot do. The profile + // and the allowed list ride along: dropped here, every action went to the + // default profile and a persona's profile limit was never kept. + const allowed = Array.isArray(frame.allowed) + ? frame.allowed.filter((a): a is string => typeof a === 'string') + : undefined; + return { + requestId: frame.requestId, + threadId: frame.threadId, + action: frame.action!, + ...(typeof frame.profile === 'string' && frame.profile ? { profile: frame.profile } : {}), + // A malformed list is still a limit: kept (empty), never widened to every profile. + ...(frame.allowed !== undefined ? { allowed: allowed ?? [] } : {}) + }; } function asHarnessRequest(data: unknown): DeviceHarnessRequest | null { diff --git a/src/desktop/quickchat/index.ts b/src/desktop/quickchat/index.ts index 96912c57..94ab3393 100644 --- a/src/desktop/quickchat/index.ts +++ b/src/desktop/quickchat/index.ts @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto'; -import { globalShortcut, ipcMain, type BrowserWindow } from 'electron'; +import { globalShortcut, ipcMain, shell, type BrowserWindow } from 'electron'; import { log } from '../../server/log'; import { handleLocal } from '../ipc-bridge'; import { @@ -21,8 +21,11 @@ import { placeHud, compactOverlay, placeOverlay, - setOverlayWorkspaceVisibility + placeOverlayBeside, + setOverlayWorkspaceVisibility, + type DockRect } from './windows'; +import { peekFrontWindow, readWindowText, stopWindowPeek, warmWindowPeek, windowBounds, windowPeekSupported } from './window-peek'; import { eventTurnId } from '../../shared/settledTurns'; import { PillTurns, type PillEventParams } from './pill-turns'; import { activityLabel } from '../../shared/activity'; @@ -35,8 +38,10 @@ import type { QuickChatSettings, QuickChatShortcutStatus, QuickChatStatus, + QuickChatWindow, StartTurnInput, - StartTurnResult + StartTurnResult, + WindowContext } from '../../shared/types'; // Quick Chat: the Spotlight-style overlay, the bottom-left status pill, and the @@ -90,6 +95,8 @@ export interface QuickChatSurface { registerIpc(): void; /** Summon or dismiss the overlay — the shortcut, the tray, the CLI, the HUD. */ toggle(): void; + /** "Ask about this window" — its shortcut, or `stem --ask-window` (the Wayland path). */ + askAboutWindow(): void; /** Is the overlay on screen? (app.on('activate') asks before reopening the app.) */ overlayVisible(): boolean; /** Apply the parts of a Quick Chat settings change no other layer can. */ @@ -100,6 +107,8 @@ export interface QuickChatSurface { sendToOverlay(channel: string, payload: unknown): void; /** Push on a channel the HUD pill renders (theme changes). No-op when it is gone. */ sendToHud(channel: string, payload: unknown): void; + /** Let go of the window helper (app quit). */ + stop(): void; /** Bring the overlay back when it owns `threadId` (approval cards). */ revealIfOwns(threadId: string | null | undefined): void; /** @@ -132,6 +141,21 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { * that does nothing. */ let desiredShortcut: string | null = null; + /** The same pair for "Ask about this window" (Mac only). */ + let currentWindowShortcut: string | null = null; + let desiredWindowShortcut: string | null = null; + + // ---- "Ask about this window" ---- + // + // The window the overlay's session was summoned for: the panel docks beside + // it (never on top when the screen has room) and follows it while shown, and + // the first prompt sent with its chip carries its text. Cleared whenever the + // session starts fresh without one. + let peek: { window: QuickChatWindow; bounds: DockRect; text: Promise } | null = null; + /** A press is still capturing: a second press waits its turn instead of racing it. */ + let peeking = false; + /** Polls the anchored window's frame while the overlay is on screen. */ + let followTimer: NodeJS.Timeout | null = null; /** Cached "show overlay on all Spaces" setting, applied once per overlay window. */ let overlayOnAllDisplays = true; /** Cached inactivity timeout (ms) after which a summon starts a fresh thread. */ @@ -334,19 +358,104 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { * answer the user re-summoned to read). The overlay's React state persists across * hide/show, so resuming needs no payload beyond the reset flag. */ - function showQuickChat(reset: boolean): void { + function showQuickChat(reset: boolean, window?: QuickChatWindow): void { const win = ensureOverlayWindow(); // Resuming puts the answer in front of the user: that's reading it. if (!reset && !overlay.turnRunning) markOverlayThreadRead(); hideHud(); deps.beginSummon(); - placeOverlay(win, reset); + if (peek) { + placeOverlayBeside(win, peek.bounds, !reset, reset); + followWindow(); + } else placeOverlay(win, reset); // Surface the panel on whatever Space the user is on and make it the key window // so it receives keystrokes (typing, Escape) — without activating Stem, so the // previous app stays active underneath and the main window is never pulled // forward. The macOS ordering is subtle; see presentOverlayWindow. presentOverlayWindow(win); - win.webContents.send('quickchat:focus', { reset }); + win.webContents.send('quickchat:focus', window ? { reset, window } : { reset }); + } + + /** + * Keep the docked panel beside its window while both are on screen: the + * window may be dragged or resized with the panel open. Stops itself once + * the overlay hides or the window closes. + */ + function followWindow(): void { + if (followTimer) return; + followTimer = setInterval(() => { + const win = overlayWindow; + const anchored = peek; + if (!win || win.isDestroyed() || !win.isVisible() || !anchored) { + stopFollowing(); + return; + } + void windowBounds(anchored.window.windowId) + .then((b) => { + if (!b || peek !== anchored || !win.isVisible() || win.isDestroyed()) return; + const p = anchored.bounds; + if (b.x === p.x && b.y === p.y && b.width === p.width && b.height === p.height) return; + anchored.bounds = b; + const expanded = win.getBounds().height > 200; + placeOverlayBeside(win, b, expanded, !expanded); + }) + .catch(() => undefined); + }, 400); + } + + function stopFollowing(): void { + if (followTimer) clearInterval(followTimer); + followTimer = null; + } + + /** + * The window shortcut: capture the window the user is in, then summon Quick + * Chat beside it with that window as a chip. Pressed again while the overlay + * is up, it dismisses like the plain shortcut. The same window again resumes + * its conversation (unless it went idle); another window starts a fresh one. + */ + async function askAboutWindow(): Promise { + if (overlayWindow && overlayWindow.isVisible()) { + dismissQuickChat(); + return; + } + if (peeking) return; + peeking = true; + try { + const peeked = await peekFrontWindow(); + if (!peeked) { + // No other window to ask about: an ordinary summon. + toggleQuickChat(); + return; + } + const sameWindow = peek?.window.windowId === peeked.window.windowId; + const reset = !sameWindow || overlay.shouldStartFresh(Date.now(), newThreadTimeoutMs); + if (reset) overlay.clearForFreshSession(); + // The text is read while the user types; the send waits for it (see runQuickChat). + const text = reset || !peek + ? peeked.window.missing.includes('accessibility') + ? Promise.resolve(null) + : readWindowText(peeked.window.windowId) + : peek.text; + peek = { window: peeked.window, bounds: peeked.bounds, text }; + showQuickChat(reset, reset ? peeked.window : undefined); + } finally { + peeking = false; + } + } + + /** What the model is told about the window a prompt was sent with. */ + async function windowContextFor(windowId: number | undefined): Promise { + const anchored = peek; + if (windowId === undefined || !anchored || anchored.window.windowId !== windowId) return undefined; + const { app, bundleId, title } = anchored.window; + // A slow app's tree must not hold the question up for long: past this the + // turn goes with the picture alone. + const text = await Promise.race([ + anchored.text, + new Promise((resolve) => setTimeout(() => resolve(null), 5_000)) + ]); + return { windowId, app, bundleId, title, text }; } /** @@ -373,6 +482,7 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { /** Hide just the overlay window (without app.hide), so the HUD can stay visible. */ function hideOverlayWindow(): void { + stopFollowing(); if (overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible()) overlayWindow.hide(); } @@ -385,10 +495,26 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { // the configured timeout => start a new thread. Clear ownership up front so the // old thread's events stop routing to the (now reset) overlay. const reset = overlay.shouldStartFresh(Date.now(), newThreadTimeoutMs); - if (reset) overlay.clearForFreshSession(); + if (reset) { + overlay.clearForFreshSession(); + // A plain summon starts a plain session: no window chip, centred bar. + peek = null; + } showQuickChat(reset); } + /** Grab `accelerator` for `handler`; the accelerator when the OS granted it, else null. */ + function registerAccelerator(accelerator: string | null, handler: () => void): string | null { + if (!accelerator) return null; + try { + if (globalShortcut.register(accelerator, handler)) return accelerator; + log('main', 'global shortcut refused by the OS', { accelerator }); + } catch (error) { + log('main', 'global shortcut registration failed', { accelerator, error: String(error) }); + } + return null; + } + /** (Re)register the global accelerator. Returns false when registration fails. */ function applyQuickChatShortcut(accelerator: string | null): boolean { desiredShortcut = accelerator; @@ -396,16 +522,21 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { globalShortcut.unregister(currentShortcut); currentShortcut = null; } - if (!accelerator) return true; - try { - const ok = globalShortcut.register(accelerator, toggleQuickChat); - if (ok) currentShortcut = accelerator; - else log('main', 'global shortcut refused by the OS', { accelerator }); - return ok; - } catch (error) { - log('main', 'global shortcut registration failed', { accelerator, error: String(error) }); - return false; + currentShortcut = registerAccelerator(accelerator, toggleQuickChat); + return !accelerator || !!currentShortcut; + } + + /** (Re)register "Ask about this window" where the helper can see other windows (Wayland binds `--ask-window` instead). */ + function applyWindowShortcut(accelerator: string | null): void { + desiredWindowShortcut = accelerator; + if (currentWindowShortcut) { + globalShortcut.unregister(currentWindowShortcut); + currentWindowShortcut = null; } + if (!windowPeekSupported) return; + currentWindowShortcut = registerAccelerator(accelerator, () => void askAboutWindow()); + // The helper's launch was most of a first press's wait: start it now. + if (currentWindowShortcut) warmWindowPeek(); } /** @@ -417,8 +548,11 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { return { accelerator: desiredShortcut, registered: !!desiredShortcut && currentShortcut === desiredShortcut, + windowAccelerator: desiredWindowShortcut, + windowRegistered: !!desiredWindowShortcut && currentWindowShortcut === desiredWindowShortcut, wayland: isWaylandSession(), - summonCommand: quickChatSummonCommand() + summonCommand: quickChatSummonCommand(), + windowSummonCommand: quickChatSummonCommand('--ask-window') }; } @@ -561,13 +695,20 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { const turnId = prompt.turnId || randomUUID(); overlayTurnId = turnId; overlay.beginTurn(Date.now()); - // Hide just the overlay (NOT app.hide — that would also hide the HUD we're - // about to show, and re-showing the HUD would reactivate the app and surface - // the main window). The overlay is a non-activating panel, so hiding it does - // not promote the main window. The HUD is non-focusable, so it never steals - // focus from whatever app the user is in. - hideOverlayWindow(); - showHud({ phase: 'working', label: 'Working…' }, 'quickchat'); + if (peek && overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible()) { + // Docked beside a window: the answer is read right there, next to what + // it is about, so the panel grows in place instead of giving way to the + // pill. (driveHud shows no pill while the overlay is visible.) + placeOverlayBeside(overlayWindow, peek.bounds, true, false); + } else { + // Hide just the overlay (NOT app.hide — that would also hide the HUD we're + // about to show, and re-showing the HUD would reactivate the app and surface + // the main window). The overlay is a non-activating panel, so hiding it does + // not promote the main window. The HUD is non-focusable, so it never steals + // focus from whatever app the user is in. + hideOverlayWindow(); + showHud({ phase: 'working', label: 'Working…' }, 'quickchat'); + } try { const continuing = overlay.owns(prompt.threadId); @@ -585,6 +726,7 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { }); } + const windowContext = await windowContextFor(prompt.windowId); // `surface` is the whole of what makes this a Quick Chat turn: the server // resolves the Quick Chat web-search toggle and the main+quickChat // instruction composition from it (see backend:startTurn). @@ -598,7 +740,8 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { serviceTier: prompt.serviceTier, format: prompt.format, surface: 'quickChat', - attachments: prompt.attachments + attachments: prompt.attachments, + ...(windowContext ? { windowContext } : {}) } satisfies StartTurnInput ])) as StartTurnResult; overlay.noteActivity(Date.now()); @@ -676,6 +819,12 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { ensureOverlayWindow(); ensureHudWindow(); applyQuickChatShortcut(settings.shortcut); + applyWindowShortcut(settings.windowShortcut); + }, + + stop() { + stopFollowing(); + stopWindowPeek(); }, registerIpc() { @@ -729,19 +878,28 @@ export function createQuickChat(deps: QuickChatDeps): QuickChatSurface { dismissQuickChat(); }); + // The window chip's "Open Settings": the Privacy pane for the missing grant. + handleLocal('quickchat:openPrivacy', (_e, which: 'screen' | 'accessibility') => { + const pane = which === 'accessibility' ? 'Privacy_Accessibility' : 'Privacy_ScreenCapture'; + void shell.openExternal(`x-apple.systempreferences:com.apple.preference.security?${pane}`); + }); + // The panel emptied while on screen (New thread, deleted thread): drop back // to the compact bar instead of leaving a tall empty card. - handleLocal('quickchat:compact', () => { - if (overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible()) compactOverlay(overlayWindow); + handleLocal('quickchat:compact', (_e, chip?: boolean) => { + if (overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible()) compactOverlay(overlayWindow, chip === true); }); }, toggle: toggleQuickChat, + askAboutWindow: () => void askAboutWindow(), + overlayVisible: () => overlayWindow?.isVisible() ?? false, applySettings(patch, next) { if ('shortcut' in patch) applyQuickChatShortcut(next.shortcut); + if ('windowShortcut' in patch) applyWindowShortcut(next.windowShortcut); if ('showOnAllDisplays' in patch) { overlayOnAllDisplays = next.showOnAllDisplays; applyOverlayWorkspaceVisibility(); diff --git a/src/desktop/quickchat/window-peek.ts b/src/desktop/quickchat/window-peek.ts new file mode 100644 index 00000000..dda09aad --- /dev/null +++ b/src/desktop/quickchat/window-peek.ts @@ -0,0 +1,130 @@ +import { screen } from 'electron'; +import { HelperProcess, resolveHelperPath } from '../computer-host/helper'; +import { log } from '../../server/log'; +import type { QuickChatWindow } from '../../shared/types'; + +// "Ask about this window": the helper calls behind the second Quick Chat +// shortcut (native/mac/stem-computer/WindowPeek.swift, and its Windows twin +// native/windows/stem-computer/Peek.cs, and on GNOME native/linux/stem-computer). +// One helper process of its own, +// started on first use and kept, so a press costs a capture and not a process +// launch; it never selects a window or touches a computer-control run. + +/** Where a helper can see other apps' windows. */ +export const windowPeekSupported = process.platform === 'darwin' || process.platform === 'win32' || process.platform === 'linux'; + +/** The window as the panel needs it, plus the frame the panel docks against. */ +export interface PeekedWindow { + window: QuickChatWindow; + bounds: Rect; +} + +export interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +/** How much of a window's text goes before the model. */ +const MAX_TEXT_CHARS = 20_000; + +let helper: HelperProcess | null = null; +let starting: Promise | null = null; + +async function ensureHelper(): Promise { + if (helper?.alive) return helper; + if (!starting) { + starting = resolveHelperPath() + .then((binary) => (helper = new HelperProcess(binary))) + .finally(() => (starting = null)); + } + return starting; +} + +/** Start the helper ahead of the first press, so that press costs a capture only. */ +export function warmWindowPeek(): void { + if (!windowPeekSupported) return; + ensureHelper().catch((error) => log('quickchat', 'window peek could not start the helper', { error: String(error) })); +} + +/** + * A rectangle as the helper sent it, in the units Electron places windows in: + * the Mac helper speaks points already; the Windows one, which is DPI aware, + * speaks physical pixels, and Electron's are device-independent. + */ +function asRect(value: unknown): Rect | null { + const r = value as Partial | null | undefined; + if (!r || [r.x, r.y, r.width, r.height].some((n) => typeof n !== 'number')) return null; + const rect = { x: r.x!, y: r.y!, width: r.width!, height: r.height! }; + return process.platform === 'win32' ? screen.screenToDipRect(null, rect) : rect; +} + +/** + * The window the person was in when they pressed the shortcut, with one + * picture of it. Null when there is no such window (or this is not a Mac); + * a missing permission comes back as `missing` with whatever could be read. + */ +export async function peekFrontWindow(): Promise { + if (!windowPeekSupported) return null; + let reply; + try { + reply = await (await ensureHelper()).call('peek-window', {}, 10_000); + } catch (error) { + log('quickchat', 'window peek could not start the helper', { error: String(error) }); + return null; + } + const raw = reply as unknown as { + ok: boolean; + error?: string; + window?: { windowId: number; app: string; bundleId: string | null; title: string; bounds: unknown } | null; + access?: { screen?: boolean; accessibility?: boolean }; + screenshot?: { jpegBase64: string }; + }; + if (!raw.ok || !raw.window) { + if (!raw.ok) log('quickchat', 'window peek failed', { error: raw.error }); + return null; + } + const bounds = asRect(raw.window.bounds); + if (!bounds) return null; + const missing: QuickChatWindow['missing'] = []; + if (raw.access?.screen === false) missing.push('screen'); + if (raw.access?.accessibility === false) missing.push('accessibility'); + return { + bounds, + window: { + windowId: raw.window.windowId, + app: raw.window.app, + bundleId: raw.window.bundleId ?? null, + title: raw.window.title, + imageBase64: raw.screenshot?.jpegBase64 ?? null, + missing + } + }; +} + +/** The window's readable text, or null when its app shows none (or Accessibility is off). */ +export async function readWindowText(windowId: number): Promise { + try { + const reply = (await (await ensureHelper()).call('peek-text', { windowId, maxChars: MAX_TEXT_CHARS }, 15_000)) as { + ok: boolean; + text?: string | null; + }; + return reply.ok && typeof reply.text === 'string' && reply.text.trim() ? reply.text : null; + } catch (error) { + log('quickchat', 'reading the window text failed', { error: String(error) }); + return null; + } +} + +/** Where the window is now, or null once it has closed. */ +export async function windowBounds(windowId: number): Promise { + if (!helper?.alive) return null; + const reply = (await helper.call('window-bounds', { windowId }, 2_000)) as unknown as { ok: boolean; bounds?: unknown }; + return reply.ok ? asRect(reply.bounds) : null; +} + +export function stopWindowPeek(): void { + helper?.kill(); + helper = null; +} diff --git a/src/desktop/quickchat/windows.ts b/src/desktop/quickchat/windows.ts index 76f010af..6b749b6a 100644 --- a/src/desktop/quickchat/windows.ts +++ b/src/desktop/quickchat/windows.ts @@ -1,5 +1,6 @@ import { BrowserWindow, screen } from 'electron'; -import { overlayOuterBounds, overlayWindowOptions, timeLocaleArguments, workspaceVisibilityOptions } from '../platform'; +import { placeOwnWindow } from '../computer-host/helper'; +import { isWaylandSession, overlayOuterBounds, overlayWindowOptions, timeLocaleArguments, workspaceVisibilityOptions } from '../platform'; import { loadRenderer, PRELOAD_SCRIPT } from '../renderer-assets'; // The two windows Quick Chat owns, and where on screen they go. Construction @@ -19,6 +20,8 @@ const QUICK_CHAT_PANEL_HEIGHT = 518; // Bottom-left status HUD pill. const HUD_WIDTH = 320; const HUD_HEIGHT = 46; +/** The pill's window title: how Stem's GNOME extension finds it to place it on Wayland. */ +const HUD_TITLE = 'Stem status'; /** * The Quick Chat overlay (Spotlight-style): a frameless, always-on-top, @@ -126,6 +129,9 @@ export function createHudWindow(deps: { installNavigationGuards(win: BrowserWind }); deps.installNavigationGuards(win); + // A fixed title, not the page's: on Wayland the pill is found by it. + win.setTitle(HUD_TITLE); + win.on('page-title-updated', (e) => e.preventDefault()); // The status pill must float above everything and follow the user across every // Space and into full-screen apps — unlike the overlay, this is unconditional @@ -158,13 +164,68 @@ export function placeOverlay(win: BrowserWindow, reset: boolean): void { ); } +/** Height the window chip adds to the compact bar ("Ask about this window"). */ +const WINDOW_CHIP_HEIGHT = 58; +/** Space between the panel and the window it docks against. */ +const DOCK_GAP = 12; + +export interface DockRect { + x: number; + y: number; + width: number; + height: number; +} + +/** + * Where the panel goes next to the window it was asked about, never on top of + * it when the screen has room: right of the window (top edges level), else + * left, else below, else above — the first spot the whole panel fits on the + * window's display. A window that leaves no room anywhere (it fills the + * screen) gets the panel over its bottom-right corner. Pure, for the tests. + */ +export function dockBeside(target: DockRect, panel: { width: number; height: number }, area: DockRect): { x: number; y: number } { + const { width: w, height: h } = panel; + const right = area.x + area.width; + const bottom = area.y + area.height; + const clamp = (v: number, lo: number, hi: number) => Math.round(Math.max(lo, Math.min(v, hi))); + const levelY = clamp(target.y, area.y, bottom - h); + const centredX = clamp(target.x + (target.width - w) / 2, area.x, right - w); + if (target.x + target.width + DOCK_GAP + w <= right) return { x: Math.round(target.x + target.width + DOCK_GAP), y: levelY }; + if (target.x - DOCK_GAP - w >= area.x) return { x: Math.round(target.x - DOCK_GAP - w), y: levelY }; + if (target.y + target.height + DOCK_GAP + h <= bottom) return { x: centredX, y: Math.round(target.y + target.height + DOCK_GAP) }; + if (target.y - DOCK_GAP - h >= area.y) return { x: centredX, y: Math.round(target.y - DOCK_GAP - h) }; + const inset = 16; + return { + x: clamp(Math.min(target.x + target.width, right) - w - inset, area.x, right - w), + y: clamp(Math.min(target.y + target.height, bottom) - h - inset, area.y, bottom - h) + }; +} + +/** + * Place the overlay beside the window it was summoned for (see dockBeside), + * on that window's display. `chip`: the compact bar carries the window chip. + */ +export function placeOverlayBeside(win: BrowserWindow, target: DockRect, expanded: boolean, chip: boolean): void { + const { workArea } = screen.getDisplayMatching({ + x: Math.round(target.x), + y: Math.round(target.y), + width: Math.max(1, Math.round(target.width)), + height: Math.max(1, Math.round(target.height)) + }); + const w = QUICK_CHAT_WIDTH; + const h = expanded ? QUICK_CHAT_PANEL_HEIGHT : QUICK_CHAT_HEIGHT + (chip ? WINDOW_CHIP_HEIGHT : 0); + const { x, y } = dockBeside(target, { width: w, height: h }, workArea); + win.setBounds(overlayOuterBounds({ x, y, width: w, height: h })); +} + /** * Shrink a visible overlay back to the compact bar where it stands (New thread, * or a deleted thread, empties the panel). Keeps the top edge, so the bar stays * where the panel's head was rather than jumping to the cursor's display. + * `chip`: the bar carries a window chip ("Ask about this window"). */ -export function compactOverlay(win: BrowserWindow): void { - const outer = overlayOuterBounds({ x: 0, y: 0, width: QUICK_CHAT_WIDTH, height: QUICK_CHAT_HEIGHT }); +export function compactOverlay(win: BrowserWindow, chip = false): void { + const outer = overlayOuterBounds({ x: 0, y: 0, width: QUICK_CHAT_WIDTH, height: QUICK_CHAT_HEIGHT + (chip ? WINDOW_CHIP_HEIGHT : 0) }); const { x, y, width } = win.getBounds(); win.setBounds({ x, y, width, height: outer.height }); } @@ -172,10 +233,9 @@ export function compactOverlay(win: BrowserWindow): void { /** Park the pill in the bottom-left of the display under the cursor. */ export function placeHud(win: BrowserWindow): void { const { workArea } = screen.getDisplayNearestPoint(screen.getCursorScreenPoint()); - win.setBounds({ - x: Math.round(workArea.x + 16), - y: Math.round(workArea.y + workArea.height - HUD_HEIGHT - 16), - width: HUD_WIDTH, - height: HUD_HEIGHT - }); + const x = Math.round(workArea.x + 16); + const y = Math.round(workArea.y + workArea.height - HUD_HEIGHT - 16); + win.setBounds({ x, y, width: HUD_WIDTH, height: HUD_HEIGHT }); + // Wayland ignores setBounds' position (GNOME centres new windows). + if (isWaylandSession()) placeOwnWindow(HUD_TITLE, x, y); } diff --git a/src/desktop/settings.ts b/src/desktop/settings.ts index fdd5bdc4..33027013 100644 --- a/src/desktop/settings.ts +++ b/src/desktop/settings.ts @@ -35,6 +35,7 @@ const DEFAULTS: ClientSettings = { quickChat: { // Unset until the user records one in Settings. shortcut: null, + windowShortcut: null, showOnAllDisplays: true, // Show the progress pill for main-window threads when the main window loses // focus (switch Spaces/apps), so an active thread stays visible. @@ -81,6 +82,7 @@ function coerceClientSettings(raw: Partial | undefined): ClientS return { quickChat: { shortcut: typeof qc.shortcut === 'string' && qc.shortcut.trim() ? qc.shortcut : null, + windowShortcut: typeof qc.windowShortcut === 'string' && qc.windowShortcut.trim() ? qc.windowShortcut : null, showOnAllDisplays: typeof qc.showOnAllDisplays === 'boolean' ? qc.showOnAllDisplays : d.quickChat.showOnAllDisplays, followAcrossSpaces: @@ -148,9 +150,10 @@ export async function readClientSettings(): Promise { export function updateClientQuickChat(patch: Partial): Promise { return updateClientDocument(async (doc) => { const cur = await migrated(doc); - // Only the three keys this side owns; the rest of the patch went to the server. + // Only the keys this side owns; the rest of the patch went to the server. const mine: Partial = {}; if ('shortcut' in patch) mine.shortcut = patch.shortcut ?? null; + if ('windowShortcut' in patch) mine.windowShortcut = patch.windowShortcut ?? null; if ('showOnAllDisplays' in patch) mine.showOnAllDisplays = patch.showOnAllDisplays; if ('followAcrossSpaces' in patch) mine.followAcrossSpaces = patch.followAcrossSpaces; doc.settings = coerceClientSettings({ ...cur, quickChat: { ...cur.quickChat, ...mine } }); diff --git a/src/desktop/updates.ts b/src/desktop/updates.ts index 4d5e4d66..2a366d1e 100644 --- a/src/desktop/updates.ts +++ b/src/desktop/updates.ts @@ -16,7 +16,8 @@ import { readClientSettings } from './settings'; // What "update" means depends on how Stem got here, and the split is decided // once, at startup: // -// auto — the AppImage, and a mac build installed in an Applications folder. +// auto — the AppImage, the Windows installer, and a mac build installed in +// an Applications folder. // electron-updater reads the feed electron-builder put beside the release // (latest-linux*.yml, latest-mac.yml) and downloads the new build on its // own. The AppImage swaps the file on restart — or silently on quit, @@ -25,7 +26,9 @@ import { readClientSettings } from './settings'; // and hands it to Squirrel.Mac, which checks that the new bundle is signed // by the same Developer ID as the running one (mac builds are signed since // 2026-09, see the `mac` block in electron-builder.yml) and replaces -// Stem.app after the app quits. +// Stem.app after the app quits. On Windows it fetches the next per-user +// NSIS installer (latest.yml) and runs it silently as the app quits; +// nothing there needs administrator rights. // // manual — the deb, and a mac build running from anywhere Squirrel cannot // write: straight off the mounted dmg, or a copy macOS translocated out of @@ -91,6 +94,9 @@ export function updateModeFor(f: UpdateModeFacts): UpdateMode { // home: an Applications folder. Off the read-only dmg, or translocated out // of Downloads, the swap could only fail — the release page still works. if (f.platform === 'darwin' && f.inApplicationsFolder) return 'auto'; + // Windows ships only the NSIS installer, which installs per user into a + // folder this user owns; electron-updater runs the next one over it. + if (f.platform === 'win32') return 'auto'; return 'manual'; } diff --git a/src/preload/dictation-capture.ts b/src/preload/dictation-capture.ts new file mode 100644 index 00000000..8f12be13 --- /dev/null +++ b/src/preload/dictation-capture.ts @@ -0,0 +1,46 @@ +// The microphone for dictation off the Mac, captured here in the preload so +// the composers keep the one start/stop API the Mac's helper answers (see +// desktop/dictation/whisper.ts). Chromium resamples the input to the 16 kHz +// mono Whisper reads, and each quarter second goes to the main process as it +// is heard. A Mac never comes here: its helper owns the microphone. + +const SAMPLE_RATE = 16_000; +/** 4096 samples at 16 kHz: a block a quarter of a second long. */ +const BLOCK = 4096; + +let stream: MediaStream | null = null; +let context: AudioContext | null = null; + +export async function startCapture(onBlock: (samples: Float32Array) => void): Promise { + stopCapture(); + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true } + }); + } catch (e) { + const name = e instanceof DOMException ? e.name : ''; + throw new Error( + name === 'NotFoundError' + ? 'No microphone is connected to this computer.' + : name === 'NotAllowedError' + ? 'Stem is not allowed to use the microphone. Allow it in the system’s privacy settings.' + : `The microphone could not be opened: ${e instanceof Error ? e.message : String(e)}` + ); + } + context = new AudioContext({ sampleRate: SAMPLE_RATE }); + const source = context.createMediaStreamSource(stream); + // A ScriptProcessor rather than an AudioWorklet: a worklet loads its code + // from a URL, which the packaged app's CSP does not allow here. + const tap = context.createScriptProcessor(BLOCK, 1, 1); + tap.onaudioprocess = (event) => onBlock(new Float32Array(event.inputBuffer.getChannelData(0))); + source.connect(tap); + // A processor runs only while connected to the output; it writes silence there. + tap.connect(context.destination); +} + +export function stopCapture(): void { + for (const track of stream?.getTracks() ?? []) track.stop(); + stream = null; + void context?.close().catch(() => undefined); + context = null; +} diff --git a/src/preload/index.ts b/src/preload/index.ts index d5b1b89c..0a26b68a 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1,4 +1,5 @@ import { contextBridge, ipcRenderer, webUtils } from 'electron'; +import { startCapture, stopCapture } from './dictation-capture'; import type { FolderSettings, ActivitySnapshot, @@ -70,12 +71,14 @@ import type { ThemeSettings, ThemeState, UpdateStatus, + MenuCommand, UpdatesSettings, RecorderState, DictationUpdate, RecordingDraft } from '../shared/types'; import { parseTimeLocaleArgs } from '../shared/time-locale'; +import type { ShareOptions, ShareSource } from '../shared/share'; const api: StemApi = { // Sandboxed preloads still see process.platform; exotic platforms never ship. @@ -192,6 +195,8 @@ const api: StemApi = { ipcRenderer.invoke('mcp:setEnabled', name, enabled), setMcpServerLocation: (name: string, deviceId: string | null) => ipcRenderer.invoke('mcp:setLocation', name, deviceId), + setMcpServerApps: (name: string, apps: string[] | null) => ipcRenderer.invoke('mcp:setApps', name, apps), + mcpServersForApp: (app: string, bundleId: string | null) => ipcRenderer.invoke('mcp:forApp', app, bundleId), loginMcpServer: (name: string) => ipcRenderer.invoke('mcp:login', name), restartRuntime: () => ipcRenderer.invoke('runtime:restart'), @@ -230,9 +235,34 @@ const api: StemApi = { return () => ipcRenderer.removeListener('recorder:openSheet', handler); }, dictationLanguages: () => ipcRenderer.invoke('dictation:languages'), - startDictation: (locale: string | null) => ipcRenderer.invoke('dictation:start', locale), - stopDictation: () => ipcRenderer.invoke('dictation:stop'), - cancelDictation: () => ipcRenderer.invoke('dictation:cancel'), + // Off a Mac the microphone is captured here and streamed to Whisper in the + // main process. Start is sent first, so the session exists before the + // first block of audio arrives; what is said while the model loads is kept. + startDictation: async (locale: string | null, model?: 'fast' | 'accurate'): Promise<{ session: number; locale: string }> => { + if (process.platform === 'darwin') return ipcRenderer.invoke('dictation:start', locale); + const started = ipcRenderer.invoke('dictation:start', locale, model ?? 'fast') as Promise<{ session: number; locale: string }>; + try { + await startCapture((samples) => void ipcRenderer.invoke('dictation:audio', samples).catch(() => undefined)); + } catch (e) { + started.catch(() => undefined); + await ipcRenderer.invoke('dictation:cancel'); + throw e; + } + try { + return await started; + } catch (e) { + stopCapture(); + throw e; + } + }, + stopDictation: () => { + stopCapture(); + return ipcRenderer.invoke('dictation:stop'); + }, + cancelDictation: () => { + stopCapture(); + return ipcRenderer.invoke('dictation:cancel'); + }, onDictationUpdate: (listener: (update: DictationUpdate) => void) => { const handler = (_e: unknown, update: DictationUpdate) => listener(update); ipcRenderer.on('dictation:update', handler); @@ -255,6 +285,8 @@ const api: StemApi = { browserHostState: () => ipcRenderer.invoke('browserHost:localState'), setBrowserHostEnabled: (enabled: boolean) => ipcRenderer.invoke('browserHost:setEnabled', enabled), chooseBrowser: (id: string) => ipcRenderer.invoke('browserHost:choose', id), + renameBrowser: (id: string, name: string) => ipcRenderer.invoke('browserHost:rename', id, name), + forgetBrowser: (id: string) => ipcRenderer.invoke('browserHost:forget', id), setUpBrowserControl: () => ipcRenderer.invoke('browserHost:setUp'), openBrowserExtensionsPage: (browserId?: string) => ipcRenderer.invoke('browserHost:openExtensions', browserId), onMcpAdminApproval: (listener: (proposal: McpAdminProposal) => void) => { @@ -429,6 +461,11 @@ const api: StemApi = { ipcRenderer.invoke('pins:update', threadId, pinId, patch), removePin: (threadId: string, pinId: string) => ipcRenderer.invoke('pins:remove', threadId, pinId), reorderPins: (threadId: string, pinIds: string[]) => ipcRenderer.invoke('pins:reorder', threadId, pinIds), + shareStatus: () => ipcRenderer.invoke('share:status'), + previewShare: (source: ShareSource, opts?: ShareOptions) => ipcRenderer.invoke('share:preview', source, opts), + createShare: (source: ShareSource, opts?: ShareOptions) => ipcRenderer.invoke('share:create', source, opts), + listShares: () => ipcRenderer.invoke('share:list'), + revokeShare: (id: string) => ipcRenderer.invoke('share:revoke', id), renameChat: (threadId: string, name: string) => ipcRenderer.invoke('chats:rename', threadId, name), deleteChat: (threadId: string) => ipcRenderer.invoke('chats:delete', threadId), createFolder: (name: string, parentId: string | null, settings?: Omit) => @@ -474,8 +511,30 @@ const api: StemApi = { addMailParticipant: (conversationId: string, personaId: string) => ipcRenderer.invoke('mail:addParticipant', conversationId, personaId), stopMail: (conversationId: string) => ipcRenderer.invoke('mail:stop', conversationId), - resolveMailApproval: (itemId: string, decision: 'allow' | 'deny') => - ipcRenderer.invoke('mail:resolveApproval', itemId, decision), + // The edited draft body rides only for a send draft (older servers reject a 3rd arg). + resolveMailApproval: (itemId: string, decision: 'allow' | 'deny', body?: string) => + body !== undefined + ? ipcRenderer.invoke('mail:resolveApproval', itemId, decision, body) + : ipcRenderer.invoke('mail:resolveApproval', itemId, decision), + composeExternalMail: (input: { to: string; subject?: string; body: string; self?: string }) => + ipcRenderer.invoke('mail:composeExternal', input), + releaseMail: (conversationId: string, itemId: string) => ipcRenderer.invoke('mail:release', conversationId, itemId), + draftExternalMail: (conversationId: string, personaId: string, instructions?: string) => + ipcRenderer.invoke('mail:draftExternal', conversationId, personaId, instructions ?? ''), + interStemStatus: () => ipcRenderer.invoke('interstem:status'), + interStemAddAddress: (address: string) => ipcRenderer.invoke('interstem:addAddress', address), + interStemVerifyAddress: (address: string, code: string) => ipcRenderer.invoke('interstem:verifyAddress', address, code), + interStemCheckLink: (address: string) => ipcRenderer.invoke('interstem:checkLink', address), + interStemRemoveAddress: (address: string) => ipcRenderer.invoke('interstem:removeAddress', address), + interStemSetUnlisted: (unlisted: boolean) => ipcRenderer.invoke('interstem:setUnlisted', unlisted), + interStemContact: (op: 'accept' | 'block' | 'unblock' | 'verify' | 'unverify' | 'ackKey', address: string) => + ipcRenderer.invoke('interstem:contact', op, address), + interStemInvite: (address: string) => ipcRenderer.invoke('interstem:invite', address), + onInterStemChanged: (listener: () => void) => { + const handler = () => listener(); + ipcRenderer.on('interstem:changed', handler); + return () => ipcRenderer.removeListener('interstem:changed', handler); + }, setMailRead: (conversationIds: string[], read: boolean) => ipcRenderer.invoke('mail:setRead', conversationIds, read), setMailArchived: (conversationIds: string[], archived: boolean) => @@ -552,6 +611,12 @@ const api: StemApi = { markReleaseNotesSeen: () => ipcRenderer.invoke('releaseNotes:markSeen'), updateReleaseNotesSettings: (patch: Partial) => ipcRenderer.invoke('settings:updateReleaseNotes', patch), + onMenuCommand: (listener: (command: MenuCommand) => void) => { + const handler = (_e: unknown, command: MenuCommand) => listener(command); + ipcRenderer.on('menu:command', handler); + return () => ipcRenderer.removeListener('menu:command', handler); + }, + setMenuChats: (threadIds: string[], titles: string[]) => ipcRenderer.invoke('menu:setChats', threadIds, titles), getUpdateStatus: () => ipcRenderer.invoke('updates:get'), checkForUpdates: () => ipcRenderer.invoke('updates:check'), installUpdate: () => ipcRenderer.invoke('updates:install'), @@ -622,7 +687,8 @@ const api: StemApi = { revealQuickChat: () => ipcRenderer.invoke('quickchat:reveal'), revealMain: () => ipcRenderer.invoke('main:reveal'), hideQuickChat: () => ipcRenderer.invoke('quickchat:hide'), - compactQuickChat: () => ipcRenderer.invoke('quickchat:compact'), + openQuickChatPrivacy: (which: 'screen' | 'accessibility') => ipcRenderer.invoke('quickchat:openPrivacy', which), + compactQuickChat: (chip?: boolean) => ipcRenderer.invoke('quickchat:compact', chip === true), onQuickChatFocus: (listener: (focus: QuickChatFocus) => void) => { const handler = (_e: unknown, focus: QuickChatFocus) => listener(focus); ipcRenderer.on('quickchat:focus', handler); diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index cb652f0f..3665cf79 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -17,16 +17,17 @@ import type { TaskNotifyPayload, TurnAttachment, ThreadStatus, - UpdateStatus + UpdateStatus, + ManageTabId } from '../shared/types'; import { AUTH_PROVIDER_IDS, providerName } from '../shared/providers'; -import { emptyInboxState, isUnread, placement, withRead, type InboxState } from '../shared/inbox'; +import { emptyInboxState, isUnread, placement, withArchived, withRead, type InboxState } from '../shared/inbox'; import { previewText } from '../shared/plain-text'; import { resendAttachments } from './attachments'; import { ChatView, type ChatViewHandle } from './chat/ChatView'; import { OnboardingGate } from './onboarding/OnboardingGate'; import { ShortcutHint, glyphsFor, useShortcut } from './shortcuts'; -import { ManagePanel } from './manage/ManagePanel'; +import { MANAGE_TAB_IDS, MANAGE_TAB_KEY, ManagePanel } from './manage/ManagePanel'; import { McpApprovalCard } from './manage/McpApprovalCard'; import { InstructionsApprovalCard } from './manage/InstructionsApprovalCard'; import { SkillApprovalCard } from './manage/SkillApprovalCard'; @@ -35,6 +36,8 @@ import { ApprovalNotice, MissedApprovalDialog } from './manage/ApprovalCard'; import { approvalsElsewhere, approvalsForThread } from './manage/approvalQueue'; import { useApprovals } from './manage/approvalStore'; import { DeleteThreadDialog } from './DeleteThreadDialog'; +import { ShareDialog } from './share/ShareDialog'; +import type { ShareSource } from '../shared/share'; import { CHATS_TAB_IDS, CHATS_TAB_KEY, type ChatsTab } from './chats/ChatList'; import { persistReturnChatId, @@ -45,7 +48,7 @@ import { } from './chats/return-chat'; import { SnoozeMenu } from './chats/SnoozeMenu'; import { hasMailWaiting, useMail } from './mail/useMail'; -import { MailComposeView, MailConversationView, type MailForwardDraft, type MailViewHandle } from './mail/MailView'; +import { MailComposeView, MailConversationView, type ExternalComposeInput, type MailForwardDraft, type MailViewHandle } from './mail/MailView'; import { ActivityIndicator } from './ui/ActivityIndicator'; import { TaskAlertModal } from './TaskAlertModal'; import { ReleaseNotesModal } from './ReleaseNotesModal'; @@ -135,6 +138,11 @@ export default function App() { // ChatList) because the open-a-chat handlers below need to know whether the // Inbox was selected at that moment — that is what pins the Return-to-chat row. const [chatsTab, setChatsTab] = useRememberedTab(CHATS_TAB_KEY, CHATS_TAB_IDS, 'inbox'); + // The right-hand panel's tab, held here rather than in ManagePanel so the View + // menu (and ⌘,) can switch it — even while the panel is hidden. + const [manageTab, setManageTab] = useRememberedTab(MANAGE_TAB_KEY, MANAGE_TAB_IDS, 'chats'); + // Public share links: one answer (turnId) or the whole chat. + const [shareTarget, setShareTarget] = useState<{ source: ShareSource; anchorId: string } | null>(null); const chatsTabRef = useRef(chatsTab); chatsTabRef.current = chatsTab; // The Return-to-chat target: the ordinary chat last opened (or created) while @@ -385,13 +393,16 @@ export default function App() { returnChatId, chats: displayList.chats, chatsLoaded, + inbox: displayList.inbox, + now: Date.now(), activeThreadId, mailPaneOpen: mailView !== null }), - [returnChatId, displayList.chats, chatsLoaded, activeThreadId, mailView] + [returnChatId, displayList.chats, displayList.inbox, chatsLoaded, activeThreadId, mailView] ); - // A target the loaded list no longer contains can't be returned to (deleted - // here or on another device) — drop it rather than keep a dead pointer. + // A target the loaded list no longer contains (deleted here or on another + // device) or that was archived can't be returned to — drop it rather than + // keep a dead pointer. useEffect(() => { if (inboxReturn.orphaned) dismissInboxReturn(); }, [inboxReturn.orphaned, dismissInboxReturn]); @@ -1111,6 +1122,10 @@ export default function App() { // ⌘N / ⌘\ — mirror the titlebar buttons. (Composer shortcuts live in ChatView.) useShortcut('new-conversation', () => newConversation()); useShortcut('toggle-inspector', () => setShowInspector((v) => !v)); + useShortcut('open-settings', () => { + setShowInspector(true); + setManageTab('settings'); + }); const openChat = useCallback( // `userNavigation` separates the user opening a chat (clicks, alerts, forks — @@ -1216,6 +1231,23 @@ export default function App() { return () => window.removeEventListener('focus', markVisibleRead); }, [displayList, activeThreadId, mutateInbox, updatedAtMap]); + // The application menu's own commands — the ones that are not a shortcut (those + // run through ShortcutsProvider): jump to a panel tab, open a ⌘1–9 row by id. + const openChatRef = useRef(openChat); + openChatRef.current = openChat; + useEffect( + () => + window.stem.onMenuCommand((command) => { + if (command.kind === 'manage-tab') { + setShowInspector(true); + setManageTab(command.tab); + } else if (command.kind === 'open-chat') { + void openChatRef.current(command.threadId); + } + }), + [setManageTab] + ); + // ---- Coming back after the event stream was away ---- // // Reconcile the running flags before refreshing the visible transcript. A @@ -1343,6 +1375,18 @@ export default function App() { }, [mutateInbox, updatedAtMap] ); + // Chat archive: out of the tree into its folded Archived section, kept whole. + // A timestamp like mail's, so a new turn in the chat brings it back by itself. + const onSetArchived = useCallback( + (threadIds: string[], archived: boolean) => { + const now = Date.now(); + mutateInbox( + (inbox) => withArchived(inbox, threadIds, archived, now), + () => window.stem.setInboxArchived(threadIds, archived) + ); + }, + [mutateInbox] + ); // ---- mail actions the rail and centre pane share ---- const composeMail = useCallback(() => setMailView({ kind: 'compose' }), []); @@ -1370,9 +1414,9 @@ export default function App() { * just sent stays in view, with the personas' progress on it, instead of * vanishing into a folded Sent section. */ const onComposeSend = useCallback( - async (input: Parameters[0]) => { + async (input: Parameters[0] | ExternalComposeInput) => { const before = new Set(mailRef.current.conversations.map((c) => c.id)); - const list = await mailApi.compose(input); + const list = 'external' in input ? await mailApi.composeExternal(input) : await mailApi.compose(input); const created = list.conversations .filter((c) => !before.has(c.id)) .sort((a, b) => b.createdAt - a.createdAt)[0]; @@ -1381,6 +1425,10 @@ export default function App() { }, [mailApi, openMail] ); + const onShareMessage = useCallback((turnId: string) => { + const threadId = activeThreadIdRef.current; + if (threadId) setShareTarget({ source: { kind: 'chat', threadId }, anchorId: turnId }); + }, []); const onWriteSubject = useCallback( (threadId: string) => { // Round-trips through a model, so this resolves in a second or two rather @@ -1451,9 +1499,8 @@ export default function App() { }); // ---- triage shortcuts ---- - // ⌘⇧A / ⌘⇧S act on the mail conversation you are reading — archive and snooze - // are mail concepts now. ⌘⇧U toggles read on whichever the centre pane holds: - // the open mail conversation, else the open chat. Registered here, not in the + // ⌘⇧A and ⌘⇧D act on whichever the centre pane holds: the open mail + // conversation, else the open chat. ⌘⇧S snoozes mail only — chats have no snooze. Registered here, not in the // list, so they survive a hidden inspector or a parked-on-Settings panel. const mailViewRef = useRef(mailView); mailViewRef.current = mailView; @@ -1462,10 +1509,19 @@ export default function App() { mailViewRef.current?.kind === 'conversation' ? mailViewRef.current.id : null; useShortcut('archive-thread', () => { - const id = openMailId(); + const mailId = openMailId(); + if (mailId) { + mailApi.archive([mailId], true); + setMailView(null); + return; + } + // The chat stays open: archiving tidies the list, it doesn't close what you + // are reading. Pressed again on an archived chat, it brings it back. + const id = activeThreadIdRef.current; if (!id) return; - mailApi.archive([id], true); - setMailView(null); + const chat = displayList.chats.find((c) => c.threadId === id); + if (!chat) return; + onSetArchived([id], placement(chat, displayList.inbox, Date.now()) === 'inbox'); }); useShortcut('snooze-thread', () => { @@ -1863,6 +1919,9 @@ export default function App() { mailApi.addParticipant(conversation.id, personaId) } onStop={() => void mailApi.stop(conversation.id)} + onShare={(itemId) => + setShareTarget({ source: { kind: 'mail', conversationId: conversation.id }, anchorId: itemId }) + } onForward={(forward) => setMailView({ kind: 'compose', forward })} /> ); @@ -1888,6 +1947,7 @@ export default function App() { onRetry={onRetry} onEdit={onEditMessage} onFork={onForkMessage} + onShare={onShareMessage} onDelete={onDeleteFromTurn} onRetryFailedSend={onRetryFailedSend} onEditFailedSend={onEditFailedSend} @@ -1921,6 +1981,8 @@ export default function App() { + {shareTarget && ( + setShareTarget(null)} + onOpenSettings={() => { + try { + localStorage.setItem('stem.settings.tab', 'features'); + } catch { + // Lands on the remembered settings tab instead. + } + setShareTarget(null); + setShowInspector(true); + setManageTab('settings'); + }} + /> + )} {pendingDelete && ( { + onSetArchived([pendingDelete.threadId], true); + setPendingDelete(null); + }} onCancel={() => setPendingDelete(null)} /> )} @@ -2086,6 +2170,7 @@ export default function App() {
+ + {onArchive && ( + + )} diff --git a/src/renderer/chat/ChatView.tsx b/src/renderer/chat/ChatView.tsx index 9973cbac..0b9bcf8e 100644 --- a/src/renderer/chat/ChatView.tsx +++ b/src/renderer/chat/ChatView.tsx @@ -24,7 +24,8 @@ import { PinOff, ChevronRight, Clock, - Lock + Lock, + Share2 } from 'lucide-react'; import type { ActivityItem, @@ -132,6 +133,8 @@ interface ChatViewProps { onEdit: (turnId: string, newText: string) => void; /** Branch the conversation into a new chat ending at this turn. */ onFork: (turnId: string) => void; + /** Publish this answer as a public link (opens the share dialog). */ + onShare?: (turnId: string) => void; /** Delete this turn and everything after it (truncate, no re-send). */ onDelete: (turnId: string) => void; /** Re-send a message whose send failed before a turn existed (startTurn rejected — @@ -405,6 +408,7 @@ export const ChatView = forwardRef(function ChatV onRetry, onEdit, onFork, + onShare, onDelete, onRetryFailedSend, onEditFailedSend, @@ -728,6 +732,13 @@ export const ChatView = forwardRef(function ChatV {m.role === 'assistant' && !isEditing && (m.sources?.length ?? 0) > 0 && ( )} + {isLive && !isEditing && !!m.content && ( + // Text written mid-turn ("I'll go through the threads…") is not the + // end: the model may still be thinking or calling tools, whose rows + // sit above the text, often scrolled away. Keep the dots at the + // bottom of the reply until the turn settles. +
{activityIndicator}
+ )} {!isEditing && m.attachments && m.attachments.length > 0 && (
{m.attachments.map((att, i) => @@ -804,6 +815,18 @@ export const ChatView = forwardRef(function ChatV ); })()} + {onShare && m.role === 'assistant' && m.turnId && m.content.trim() && ( + + + + )} {m.role !== 'system' && m.turnId && ( <> diff --git a/src/renderer/chat/StreamingMdxView.tsx b/src/renderer/chat/StreamingMdxView.tsx index 36ba8ec7..b073853d 100644 --- a/src/renderer/chat/StreamingMdxView.tsx +++ b/src/renderer/chat/StreamingMdxView.tsx @@ -1,6 +1,8 @@ import { Fragment, memo, useRef, type ReactNode } from 'react'; -import { provisionalTail, renderMdx, splitStreamBlocks, tryRenderMdx } from '../mdx/render'; -import { MdxPlaceholder } from '../mdx/components'; +import { provisionalTail, renderMdx, splitStreamBlocks, streamingData, tryRenderMdx } from '../mdx/render'; +import { MdxPlaceholder, componentMap } from '../mdx/components'; +import { completeRows } from '../mdx/data'; +import { Diagram } from '../mdx/Diagram'; // Incremental renderer for a still-streaming reply. MdxView re-parses the entire // accumulated text on every delta — O(n) per token, O(n²) per reply. Here the text @@ -13,7 +15,9 @@ import { MdxPlaceholder } from '../mdx/components'; // which component the tail is still inside. Prose containers (Callout, Steps, // Tabs…) render live, closed provisionally; anything whose content is data // (Chart, DataTable, Quiz, Form…) shows a placeholder until its closing tag -// arrives. A half-written tag is never shown as text. +// arrives. A half-written tag is never shown as text. Diagrams and the JSON +// data components are the exception: a Diagram draws line by line, and a +// Chart, DataTable, Stats or Compare row by row, as their data arrives. export const StreamingMdxView = memo(function StreamingMdxView({ text }: { text: string }) { const cache = useRef>([]); const blocks = splitStreamBlocks(text); @@ -35,7 +39,9 @@ export const StreamingMdxView = memo(function StreamingMdxView({ text }: { text: if (cache.current.length > stableCount) cache.current.length = stableCount; let tail: ReactNode = null; const last = blocks[blocks.length - 1]; - if (last) { + const data = last && streamingData(last); + if (data) tail = {liveData(data)}; + else if (last) { const { live, pending } = provisionalTail(last); let liveNode: ReactNode = null; let waiting = pending; @@ -59,3 +65,11 @@ export const StreamingMdxView = memo(function StreamingMdxView({ text }: { text:
); }); + +/** A data component drawn from what has arrived so far; a placeholder until its first row. */ +function liveData({ name, props, value, closed }: NonNullable>): ReactNode { + // A closed fence's last line is complete, so it counts. + if (name === 'Diagram') return ; + const rows = closed ? value : completeRows(value); + return rows ? componentMap[name](props, null, { value: rows }) : ; +} diff --git a/src/renderer/chat/dictation.tsx b/src/renderer/chat/dictation.tsx index e91dcfa5..701cadb5 100644 --- a/src/renderer/chat/dictation.tsx +++ b/src/renderer/chat/dictation.tsx @@ -2,13 +2,31 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { Loader2, Mic, Square } from 'lucide-react'; import type { DictationLanguages } from '../../shared/types'; -// Dictation into a text field, on the Mac's own on-device speech model (see -// desktop/dictation.ts). The words land in the field as they are heard: what +// Dictation into a text field, on the computer's own on-device speech model +// (Apple's on a Mac, Whisper elsewhere; see desktop/dictation). The words land in the field as they are heard: what // was there before stays, and the live tail is rewritten in place until the // model settles it. Typing in the field while dictating stops the dictation, // so a hand edit is never overwritten by the next update. const LOCALE_KEY = 'stem.dictationLocale'; +const MODEL_KEY = 'stem.dictationModel'; + +/** Windows and Linux: which Whisper model listens. A per-machine choice too. */ +export function dictationModel(): 'fast' | 'accurate' { + try { + return localStorage.getItem(MODEL_KEY) === 'accurate' ? 'accurate' : 'fast'; + } catch { + return 'fast'; + } +} + +export function setDictationModel(model: 'fast' | 'accurate'): void { + try { + localStorage.setItem(MODEL_KEY, model); + } catch { + // Unsaved; the next dictation uses the fast model. + } +} /** The chosen language, or null for the Mac's own. A per-machine choice, so localStorage. */ export function dictationLocale(): string | null { @@ -130,7 +148,7 @@ export function useDictation(value: string, setValue: (text: string) => void): D setError(null); move('starting'); window.stem - .startDictation(dictationLocale()) + .startDictation(dictationLocale(), dictationModel()) .then((s) => { if (phaseRef.current !== 'starting' && phaseRef.current !== 'downloading') return; session.current = s.session; diff --git a/src/renderer/chat/tips.ts b/src/renderer/chat/tips.ts index fd1a24e6..7a203485 100644 --- a/src/renderer/chat/tips.ts +++ b/src/renderer/chat/tips.ts @@ -117,7 +117,7 @@ export const TIPS: Tip[] = [ }, { id: 'archive', - text: 'Archive a chat from its hover actions to clear the Inbox without deleting anything.' + text: 'Right-click a chat and archive it to clear it out of the list without deleting anything — it waits under Archived at the bottom.' }, { id: 'unread', diff --git a/src/renderer/chats/ChatList.tsx b/src/renderer/chats/ChatList.tsx index 7af3bf45..d6bd4d01 100644 --- a/src/renderer/chats/ChatList.tsx +++ b/src/renderer/chats/ChatList.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'; import { + Archive, CheckCheck, ChevronRight, Clock, @@ -23,7 +24,7 @@ import type { Persona, ThreadStatus } from '../../shared/types'; -import { isUnread } from '../../shared/inbox'; +import { isUnread, placement } from '../../shared/inbox'; import { useOffline } from '../hooks/useServerReachable'; import { stripCiteMarkers } from '../../shared/citations'; import { Kbd, glyphsFor, runKeyGlyphs, useShortcut, useShortcutHintMode, type ShortcutId } from '../shortcuts'; @@ -66,8 +67,10 @@ export interface ChatListProps { onRenameChat: (threadId: string, name: string) => void; onDeleteChat: (threadId: string) => void; onMoveChat: (threadId: string, folderId: string | null) => void; - /** Chat read/unread (the tree's bolding). Archive/snooze are mail concepts now. */ + /** Chat read/unread (the tree's bolding). */ onSetRead: (threadIds: string[], read: boolean) => void; + /** Archive (or bring back) chats: out of the tree, into the folded Archived section. */ + onSetArchived: (threadIds: string[], archived: boolean) => void; /** Have a model write this thread's subject from its conversation, now. */ onWriteSubject: (threadId: string) => void; @@ -210,6 +213,7 @@ export function ChatList(props: ChatListProps) { const [picked, setPicked] = useState>(() => new Set()); const anchorRef = useRef(null); const [confirmBulkDelete, setConfirmBulkDelete] = useState(false); + const [archivedOpen, setArchivedOpen] = useState(false); // ---- search ---- // The search box is collapsed to a header icon by default so it costs no vertical @@ -327,17 +331,31 @@ export function ChatList(props: ChatListProps) { return next; }); + // Archived chats leave the tree for the folded section at its foot. The same + // rule the phone uses (ios ChatListStore.isArchived): anything not in the + // Inbox placement — a leftover snooze from the old chat Inbox counts too — + // and new activity in the chat lifts it back out on its own. + const { treeChats, archivedChats } = useMemo(() => { + const now = Date.now(); + const tree: ChatSummary[] = []; + const archived: ChatSummary[] = []; + for (const c of data.chats) (placement(c, data.inbox, now) === 'inbox' ? tree : archived).push(c); + return { treeChats: tree, archivedChats: archived }; + }, [data.chats, data.inbox]); + const archivedIds = useMemo(() => new Set(archivedChats.map((c) => c.threadId)), [archivedChats]); + const childFolders = (parentId: string | null): Folder[] => data.folders.filter((f) => f.parentId === parentId).sort((a, b) => a.order - b.order); const folderChats = (folderId: string | null): ChatSummary[] => - data.chats.filter((c) => c.folderId === folderId); + treeChats.filter((c) => c.folderId === folderId); // ⌘1…⌘9 open the first nine chat rows exactly as the Chats tab shows them, top // to bottom — a collapsed folder's chats are not on screen, so they are not // counted. The same walk the tree renders with, so the number on a row's hint // and the key that opens it can never disagree. - // The same walk orders a Shift-click range. - const visibleChatIds = useMemo(() => { + // The same walk orders a Shift-click range, which may run on into the open + // Archived section; the numbers stay on the tree. + const treeChatIds = useMemo(() => { const out: string[] = []; const walk = (parentId: string | null) => { for (const f of childFolders(parentId)) if (expanded.has(f.id)) walk(f.id); @@ -346,8 +364,12 @@ export function ChatList(props: ChatListProps) { walk(null); return out; // eslint-disable-next-line react-hooks/exhaustive-deps -- childFolders/folderChats are plain views over these - }, [data.chats, data.folders, expanded]); - const numberedChatIds = useMemo(() => visibleChatIds.slice(0, 9), [visibleChatIds]); + }, [treeChats, data.folders, expanded]); + const visibleChatIds = useMemo( + () => (archivedOpen ? [...treeChatIds, ...archivedChats.map((c) => c.threadId)] : treeChatIds), + [treeChatIds, archivedChats, archivedOpen] + ); + const numberedChatIds = useMemo(() => treeChatIds.slice(0, 9), [treeChatIds]); // ---- multi-selection ---- const clearPicked = useCallback(() => { @@ -355,19 +377,19 @@ export function ChatList(props: ChatListProps) { anchorRef.current = null; }, []); // The selection belongs to the tree as it is on screen: leaving the tab or - // searching drops it, and a chat that went away (deleted, here or elsewhere) - // leaves it. + // searching drops it, and a chat that went off screen (deleted here or + // elsewhere, archived, or folded away with the Archived section) leaves it. useEffect(() => { if (tab !== 'chats' || results !== null) clearPicked(); }, [tab, results, clearPicked]); useEffect(() => { setPicked((prev) => { if (!prev.size) return prev; - const live = new Set(data.chats.map((c) => c.threadId)); + const live = new Set(visibleChatIds); const next = new Set([...prev].filter((id) => live.has(id))); return next.size === prev.size ? prev : next; }); - }, [data.chats]); + }, [visibleChatIds]); useEffect(() => { if (!picked.size) return; const onKey = (e: KeyboardEvent) => { @@ -433,6 +455,20 @@ export function ChatList(props: ChatListProps) { const id = numberedChatIds[Number(e.key) - 1]; if (id) onOpen(id); }); + // The Window menu lists the same rows under the same keys, and only while they + // are live — a menu entry for a number the list is not showing would open a + // chat the key itself refuses to. + const menuIds = numbersLive ? numberedChatIds : []; + const menuTitles = menuIds.map((id) => { + const chat = data.chats.find((c) => c.threadId === id); + return chat ? (chat.subject ?? chat.title) : ''; + }); + const menuKey = JSON.stringify([menuIds, menuTitles]); + useEffect(() => { + const [ids, titles] = JSON.parse(menuKey) as [string[], string[]]; + void window.stem.setMenuChats(ids, titles); + }, [menuKey]); + useEffect(() => () => void window.stem.setMenuChats([], []), []); const rowNumber = (threadId: string): number | null => { if (!hintMode || !numbersLive) return null; const i = numberedChatIds.indexOf(threadId); @@ -445,7 +481,7 @@ export function ChatList(props: ChatListProps) { const folderUnreadIds = useMemo(() => { const parents = new Map(data.folders.map((f) => [f.id, f.parentId])); const ids = new Map(); - for (const chat of data.chats) { + for (const chat of treeChats) { if (!chat.folderId) continue; if (!isUnread(chat, data.inbox, props.statuses[chat.threadId] === 'running')) continue; for (let id: string | null = chat.folderId; id != null; id = parents.get(id) ?? null) { @@ -455,7 +491,7 @@ export function ChatList(props: ChatListProps) { } } return ids; - }, [data.chats, data.folders, data.inbox, props.statuses]); + }, [treeChats, data.folders, data.inbox, props.statuses]); const folderUnread = useMemo( () => new Map([...folderUnreadIds].map(([id, list]) => [id, list.length])), [folderUnreadIds] @@ -720,12 +756,27 @@ export function ChatList(props: ChatListProps) { )} + {!isEditing && archivedIds.has(hit.threadId) && ( + + + + )}
); }); }; const isEmpty = data.chats.length === 0 && data.folders.length === 0; + const allPickedArchived = () => { + const ids = pickedIds(); + return ids.length > 0 && ids.every((id) => archivedIds.has(id)); + }; + const archivePicked = () => { + const ids = pickedIds(); + const restore = allPickedArchived(); + clearPicked(); + props.onSetArchived(ids, !restore); + }; const showingSearch = searching || results !== null; // Render-time clock for the tree's date buckets; the mail list keeps its own // wake-timer clock (a snoozed conversation returns on its own schedule). @@ -871,6 +922,8 @@ export function ChatList(props: ChatListProps) { const r = e.currentTarget.getBoundingClientRect(); setMenu({ kind: 'move', id: '', x: r.left, y: r.bottom + 4 }); }} + allArchived={allPickedArchived()} + onArchive={archivePicked} onDelete={() => setConfirmBulkDelete(true)} onClear={clearPicked} /> @@ -910,6 +963,15 @@ export function ChatList(props: ChatListProps) { } return rows; })()} + {archivedChats.length > 0 && ( + <> + + {archivedOpen && archivedChats.map((c) => renderChat(c, 0))} + + )} ) : ( Mark as unread + +
+ diff --git a/src/renderer/chats/return-chat.ts b/src/renderer/chats/return-chat.ts index e8f46584..1300cf84 100644 --- a/src/renderer/chats/return-chat.ts +++ b/src/renderer/chats/return-chat.ts @@ -1,3 +1,4 @@ +import { placement, type InboxState } from '../../shared/inbox'; import type { ChatSummary } from '../../shared/types'; /** @@ -8,8 +9,8 @@ import type { ChatSummary } from '../../shared/types'; * Explicit is the contract: the id is written only by that open/create rule, * never guessed from recency or activity, and it never expires on its own. It * leaves in exactly three ways — replaced by another chat opened the same way, - * dismissed, or orphaned (its chat deleted or gone from the list). Mail triage, - * tab switches, reloads and restarts all leave it alone. + * dismissed, or orphaned (its chat deleted, archived, or gone from the list). + * Mail triage, tab switches, reloads and restarts all leave it alone. */ export const RETURN_CHAT_KEY = 'stem.inbox.returnChatId'; @@ -69,14 +70,16 @@ export type ReturnChatRow = { export type ReturnChatResolution = { /** null = nothing to render (no target, list not loaded yet, or orphaned). */ row: ReturnChatRow | null; - /** The list has loaded and the target isn't in it — the caller should clear the id. */ + /** The list has loaded and the target is gone from it or archived — the caller should clear the id. */ orphaned: boolean; }; /** - * Resolve the persisted id against the chat list. Membership is the only test: - * a listed chat is an openable chat, whatever its inbox placement — archiving - * never orphans the shortcut. Before the first list arrives nothing is known, + * Resolve the persisted id against the chat list. A chat missing from the list + * or archived is orphaned: archiving says you're done with it, and a row that + * kept pointing at it would read "Currently viewing" over a chat that has left + * the tree. Snoozed is "not now", not done, so it keeps the row. Before the + * first list arrives nothing is known, * so the row stays hidden and the id is NOT judged orphaned (clearing on a * still-empty startup list would erase the very persistence being promised). */ @@ -84,15 +87,17 @@ export function resolveReturnChat(args: { returnChatId: string | null; chats: readonly ChatSummary[]; chatsLoaded: boolean; + inbox: InboxState; + now: number; activeThreadId: string | null; /** Centre pane currently shows mail (open conversation or compose form). */ mailPaneOpen: boolean; }): ReturnChatResolution { - const { returnChatId, chats, chatsLoaded, activeThreadId, mailPaneOpen } = args; + const { returnChatId, chats, chatsLoaded, inbox, now, activeThreadId, mailPaneOpen } = args; if (!returnChatId) return { row: null, orphaned: false }; if (!chatsLoaded) return { row: null, orphaned: false }; const chat = chats.find((c) => c.threadId === returnChatId); - if (!chat) return { row: null, orphaned: true }; + if (!chat || placement(chat, inbox, now) === 'archived') return { row: null, orphaned: true }; return { row: { threadId: chat.threadId, diff --git a/src/renderer/mail/MailList.tsx b/src/renderer/mail/MailList.tsx index 871f496e..26df08f8 100644 --- a/src/renderer/mail/MailList.tsx +++ b/src/renderer/mail/MailList.tsx @@ -18,6 +18,7 @@ import { mailPreviewText } from '../../shared/mail-subject'; import { SnoozeMenu } from '../chats/SnoozeMenu'; import { hasMailWaiting, personaName } from './useMail'; import { readComposeDraft, subscribeMailDrafts } from './mail-drafts'; +import { useInterStem } from './useInterStem'; // The Inbox tab's list: mail conversations, email-style. An unsent New mail // draft sits first, then the mail waiting on you (unread bold), then Working @@ -59,6 +60,7 @@ export function MailList(props: MailListProps) { const [archivedOpen, setArchivedOpen] = useState(false); const [sentOpen, setSentOpen] = useState(false); const draft = useSyncExternalStore(subscribeMailDrafts, readComposeDraft); + const interStem = useInterStem(); // One timer for the earliest snooze wake — the ChatList treatment, verbatim. const [now, setNow] = useState(() => Date.now()); @@ -81,6 +83,11 @@ export function MailList(props: MailListProps) { }, [menu]); const sections = useMemo(() => { + // Inter-Stem: first mail from someone not yet accepted waits in Requests, + // out of the Inbox, until the user accepts or blocks them. + const isRequest = (c: MailConversation) => + !!c.external && interStem.status?.contacts.find((x) => x.address === c.external!.address)?.state === 'request'; + const requests: MailConversation[] = []; const inbox: MailConversation[] = []; const working: MailConversation[] = []; const snoozed: MailConversation[] = []; @@ -95,6 +102,7 @@ export function MailList(props: MailListProps) { // until new mail addressed to the user lands, like email. A stopped // conversation surfaces here too: no reply ever lands to pull it in, and // the Inbox row is where the abort shows. + else if (isRequest(c) && hasMailWaiting(c)) requests.push(c); else if (hasMailWaiting(c) || c.status === 'aborted') inbox.push(c); // Sent and still being worked: shown above the Inbox so a fresh send // doesn't vanish into the folded Sent section while the personas work. @@ -117,8 +125,9 @@ export function MailList(props: MailListProps) { const sent = mail.conversations .filter((c) => lastSent.has(c.id)) .sort((a, b) => (lastSent.get(b.id) ?? 0) - (lastSent.get(a.id) ?? 0)); - return { inbox, working, snoozed, archived, sent }; - }, [mail, now]); + requests.sort((a, b) => b.updatedAt - a.updatedAt); + return { requests, inbox, working, snoozed, archived, sent }; + }, [mail, now, interStem.status]); /** The two-line body under a subject: the latest mail addressed to the user, * else the user's own latest send (a conversation still being worked). */ @@ -177,7 +186,7 @@ export function MailList(props: MailListProps) { - {fromLabel(c.participants)} + {c.external ? c.external.address : fromLabel(c.participants)} {c.status === 'awaiting-user' && needs you} {c.status === 'failed' && failed} {c.status === 'aborted' && · stopped} @@ -312,6 +321,12 @@ export function MailList(props: MailListProps) { {sections.inbox.map((c) => renderRow(c, 'inbox'))} {grouped &&
Working ({sections.working.length})
} {sections.working.map((c) => renderRow(c, 'working'))} + {sections.requests.length > 0 && ( +
+ Requests ({sections.requests.length}) +
+ )} + {sections.requests.map((c) => renderRow(c, 'inbox'))} {section('Snoozed', null, sections.snoozed, snoozedOpen, () => setSnoozedOpen((v) => !v), 'snoozed')} {section('Archived', null, sections.archived, archivedOpen, () => setArchivedOpen((v) => !v), 'archived')} {section( diff --git a/src/renderer/mail/MailView.tsx b/src/renderer/mail/MailView.tsx index abc67314..9abfd4cb 100644 --- a/src/renderer/mail/MailView.tsx +++ b/src/renderer/mail/MailView.tsx @@ -20,16 +20,26 @@ import { Plus, RotateCcw, Send, + Share2, ShieldAlert, Square, X, Pencil, - Trash2 + Trash2, + AtSign, + Eye, + KeyRound, + UserCheck, + UserX, + UserPlus, + Sparkles } from 'lucide-react'; import { MAIL_BETA_TITLE } from '../chats/ChatList'; +import { isExternalAddress } from '../../shared/types'; import type { MailApproval, MailComposeInput, + MailExternal, MailConversation, MailItem, MailWorkGroup, @@ -61,6 +71,25 @@ import { MailWork } from './MailWork'; import { GeneratedImages } from '../chat/GeneratedImage'; import { DictateButton, useDictation } from '../chat/dictation'; import { plainLine } from '../../shared/plain-text'; +import { useInterStem } from './useInterStem'; + +/** Compose to another Stem's owner: an email address instead of personas. */ +export interface ExternalComposeInput { + external: true; + to: string; + subject?: string; + body: string; +} + +const EXTERNAL_STATUS: Record, string> = { + queued: 'Sending…', + sent: 'Sent', + delivered: 'Delivered', + expired: 'Not collected in 7 days', + failed: 'Not sent', + unreachable: 'Can’t be reached', + 'key-changed': 'Waiting — their key changed' +}; // The centre pane's mail surface: a conversation read like the Inbox (newest // on top, the reply box above the mail it answers, older mails folded to one @@ -201,6 +230,8 @@ export const MailConversationView = forwardRef void; /** Open the compose form forwarding one of this conversation's mails. */ onForward: (draft: MailForwardDraft) => void; + /** Publish a reply (or the whole thread) as a public link: opens the share dialog. */ + onShare?: (itemId: string) => void; /** * The serving system's version (MailListResult.sys). A mail stamped with a * different one was made by older persona / skills / memory code and says so, @@ -208,7 +239,20 @@ export const MailConversationView = forwardRef(function MailConversationView( - { conversation, items, personas, onReply, pending, onRetrySend, onDropSend, onAddParticipant, onStop, onForward, currentSys }, + { + conversation, + items, + personas, + onReply, + pending, + onRetrySend, + onDropSend, + onAddParticipant, + onStop, + onForward, + onShare, + currentSys + }, ref ) { // Keyed by conversation in App, so this mounts fresh per conversation and @@ -240,10 +284,21 @@ export const MailConversationView = forwardRef (id === 'user' ? 'You' : personaName(personas, id)); - const lead = conversation.participants[0]; + // Inter-Stem: the conversation is with another Stem's owner; replies go to + // them, and personas only ever draft. + const ext = conversation.external; + const interStem = useInterStem(); + const counterpart = ext ? interStem.contact(ext.address) : undefined; + const lead = ext ? ext.address : conversation.participants[0]; // A run parked on the user's Allow/Deny takes the next mail instead of the lead. - const parkedBy = layout.entries.find((e) => e.item.approval?.status === 'pending')?.item.from; + const parkedBy = ext ? undefined : layout.entries.find((e) => e.item.approval?.status === 'pending')?.item.from; const recipient = parkedBy ?? lead; + const held = mails.filter((m) => m.external?.held); + const [actionError, setActionError] = useState(null); + const act = (fn: () => Promise) => { + setActionError(null); + fn().catch((e) => setActionError(e instanceof Error ? e.message.replace(/^Error invoking remote method '[^']+': (Error: )?/, '') : String(e))); + }; // Land at the newest mail (the top) on open and when one arrives. useEffect(() => { scrollRef.current?.scrollTo({ top: 0 }); @@ -405,6 +460,7 @@ export const MailConversationView = forwardRef{mailPreviewText(m.body) || (m.attachments?.length ? 'Attachments' : '')} {exchangeChip(m.id, entry.exchange, m.from)} {m.stale && ↩ earlier mail} + {m.statusReply && live status} {formatAt(m.at, now)} {exchangeMails(m.id, entry.exchange)} @@ -414,6 +470,9 @@ export const MailConversationView = forwardRef { const m = entry.item; + // An external conversation's own sends have no persona work behind them; + // only real runs (a persona drafting) are worth a Work card there. + const work = ext ? entry.work.filter((g) => g.runs.length > 0) : entry.work; return (
@@ -436,16 +495,58 @@ export const MailConversationView = forwardRef )} + {m.statusReply && ( + + live status + + )} + {m.statusQuery && ( + + answered on the spot + + )} + {m.external?.status && ( + + {EXTERNAL_STATUS[m.external.status]} + + )} + {m.external?.keyChanged && ( + + new key + + )} {formatAt(m.at, now)} - + {/* Only a persona's reply in the user's own thread: another person's + mail isn't ours to publish, and the memory check is the user's memory. */} + {onShare && !ext && !conversation.owner && m.from !== 'user' && !m.approval && ( + + )} + {/* Forwarding would carry another Stem's mail past the external gate. */} + {!ext && ( + + )} {m.approval?.status !== 'pending' && ( +
+ )} + {m.external?.error && m.external.status !== 'sent' && m.external.status !== 'delivered' && ( +

+ {m.external.error} + {m.external.status === 'unreachable' && ( + + )} +

+ )} + {m.approval?.kind === 'send' ? null : m.from === 'user' || isExternalAddress(m.from) ? (

{m.body}

) : ( @@ -504,10 +636,10 @@ export const MailConversationView = forwardRef )} - {(entry.exchange.length > 0 || entry.work.length > 0) && ( + {(entry.exchange.length > 0 || work.length > 0) && (
{exchangeChip(m.id, entry.exchange, m.from)} - {entry.work.map((group) => ( + {work.map((group) => ( ))}
@@ -537,7 +669,7 @@ export const MailConversationView = forwardRef m.from !== 'user')?.id; - const showOthers = conversation.participants.filter((p) => p !== recipient); + const showOthers = ext ? [] : conversation.participants.filter((p) => p !== recipient); return ( @@ -547,11 +679,21 @@ export const MailConversationView = forwardRef

{conversation.subject}

- - {name(lead)} - {conversation.participants.length > 1 && ' leads'} - {conversation.participants.slice(1).map((p) => ` · ${name(p)}`)} - + {ext ? ( + + {ext.address} + · {counterpart?.verifiedByUser ? 'verified by you' : 'verified by email'} + {conversation.participants.length > 0 && ( + · drafts by {conversation.participants.map(name).join(', ')} + )} + + ) : ( + + {name(lead)} + {conversation.participants.length > 1 && ' leads'} + {conversation.participants.slice(1).map((p) => ` · ${name(p)}`)} + + )} {working && ( }
- {addable.length > 0 && ( + {!ext && addable.length > 0 && ( + + + )} + {ext && counterpart?.keyChanged && ( +
+ + {ext.address}’s key changed — a new device, or someone else. Check with them another way + before you send anything. + + +
+ )} + {actionError &&

{actionError}

} + {!ext && addingTo && addable.length > 0 && (
{addable.map((p) => ( @@ -612,6 +777,7 @@ export const MailConversationView = forwardRef + {ext && 0} working={working} />} {[...pending].reverse().map(pendingCard)} {working && ( Promise; + onCompose: (input: MailComposeInput | ExternalComposeInput) => Promise; onCancel: () => void; /** Open as a forward: the subject prefilled, the original quoted under the note. */ forward?: MailForwardDraft; }>(function MailComposeView({ personas, onCompose, onCancel, forward }, ref) { + // Inter-Stem: writing to a person (their email address, on their Stem) + // instead of to personas. Offered once one of the user's addresses is verified. + const interStem = useInterStem(); + const canWriteOut = !forward && !!interStem.status?.addresses.some((a) => a.state === 'verified'); + const [outTo, setOutTo] = useState(null); + const writingOut = outTo !== null; // A plain New mail resumes the saved draft; a forward starts from its quote // and is never saved (it is one click away on the original mail). // A blank one is addressed to whoever the last mail went to. @@ -881,9 +1053,10 @@ export const MailComposeView = forwardRef { - if (forward || sentRef.current) return; + // The saved draft is persona mail; one to a person isn't kept (yet). + if (forward || sentRef.current || writingOut) return; writeComposeDraft({ to, subject, body, private: isPrivate, attachments: files.attachments }); - }, [forward, to, subject, body, isPrivate, files.attachments]); + }, [forward, to, subject, body, isPrivate, files.attachments, writingOut]); // A forward always has something to send — the quote — even with no note. const empty = !forward && !body.trim() && !files.attachments.length; const hasDraft = !forward && (!!body.trim() || !!subject.trim() || files.attachments.length > 0); @@ -895,6 +1068,22 @@ export const MailComposeView = forwardRef { + if (writingOut) { + if (sending || !body.trim() || !outTo.includes('@')) return; + dictation.freeze(); + setSending(true); + setError(null); + try { + sentRef.current = true; + await onCompose({ external: true, to: outTo.trim(), subject, body }); + writeComposeDraft(null); + } catch (err) { + sentRef.current = false; + setError(err instanceof Error ? err.message.replace(/^Error invoking remote method '[^']+': (Error: )?/, '') : String(err)); + setSending(false); + } + return; + } if (sending || empty || to.length === 0) return; dictation.freeze(); setSending(true); @@ -943,6 +1132,18 @@ export const MailComposeView = forwardRef Draft saved )} + {canWriteOut && ( + + )} + {!writingOut && ( + )}