From df8c454e22f4f42c9d36da40ea1b8e5dbdd2e0c6 Mon Sep 17 00:00:00 2001 From: Brent Rager Date: Mon, 7 Sep 2026 21:31:28 -0400 Subject: [PATCH] th-1fca98: render cross-client images + bump engine to persist them A photo sent from iOS showed only as text in the desktop app's view of the same conversation. The engine now persists a user turn's images as image content items (smooth-operator #564); smooth-web renders them from history. - History parsing extracted to a pure, unit-tested history.ts: historyImages() turns persisted `type:'image'` items into renderable attachments (App.tsx's live-send path already rendered attachments). operator.ts can't be node-tested (Vite-only extensionless imports), so the pure logic lives where the test runner can load it. - Engine bump: smooth-operator-server/svc rev b6c6b84 -> 9b30ed7b (carries the persistence fix), moving core 1.7.10 -> 1.10.0. That core adds three additive AgentEvent::Completed fields (spend taint flags + response id, th-126fe6) and makes Session.agent_id optional; construction sites take the serde defaults older output already produced (no behavior change). Verified: full workspace build + clippy --all-targets clean; smooth-cast /code/daemon test suites green (776 tests); web history.test.ts green; vite build clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01HVAvzYG7unJjCD9c3B9j1f --- .changeset/render-cross-client-images.md | 9 +++ Cargo.lock | 20 +++---- Cargo.toml | 6 +- crates/smooth-cast/src/coding_workflow.rs | 7 +++ crates/smooth-code/src/app.rs | 6 ++ crates/smooth-daemon/src/operator_storage.rs | 2 +- crates/smooth-web/web/src/history.test.ts | 41 +++++++++++++ crates/smooth-web/web/src/history.ts | 61 ++++++++++++++++++++ crates/smooth-web/web/src/operator.ts | 36 +----------- 9 files changed, 141 insertions(+), 47 deletions(-) create mode 100644 .changeset/render-cross-client-images.md create mode 100644 crates/smooth-web/web/src/history.test.ts create mode 100644 crates/smooth-web/web/src/history.ts diff --git a/.changeset/render-cross-client-images.md b/.changeset/render-cross-client-images.md new file mode 100644 index 000000000..932486252 --- /dev/null +++ b/.changeset/render-cross-client-images.md @@ -0,0 +1,9 @@ +--- +'@smooai/smooth': patch +--- + +th-1fca98: render images another client attached, and bump the engine to persist them. + +A photo sent from the iOS app showed only as text in the desktop app's view of the same conversation. The engine now persists a user turn's images as `image` content items (smooth-operator #564), and `smooth-web` renders them from history: history parsing moved to a pure, unit-tested `history.ts` that turns persisted `image` items into renderable attachments (the composer's live-send path already showed them). + +Engine bump: `smooth-operator-server`/`svc` git rev `b6c6b84` → `9b30ed7b` (includes the image-persistence fix), which moves core `1.7.10` → `1.10.0`. That core carries three additive `AgentEvent::Completed` fields (spend taint flags + response id, th-126fe6) and makes `Session.agent_id` optional; the few construction sites were updated to the serde defaults older output already produced (no behavior change). diff --git a/Cargo.lock b/Cargo.lock index d3902e32f..a3b401f60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5318,8 +5318,8 @@ dependencies = [ [[package]] name = "smooai-smooth-operator" -version = "1.44.1" -source = "git+https://github.com/SmooAI/smooth-operator.git?rev=b6c6b84#b6c6b848c2fdd5af362383bf0e8ef072900a0575" +version = "1.58.10" +source = "git+https://github.com/SmooAI/smooth-operator.git?rev=9b30ed7b#9b30ed7b96817f6a721584c11eee5c5b515ed4d0" dependencies = [ "anyhow", "async-trait", @@ -5350,8 +5350,8 @@ dependencies = [ [[package]] name = "smooai-smooth-operator-adapter-memory" -version = "1.44.1" -source = "git+https://github.com/SmooAI/smooth-operator.git?rev=b6c6b84#b6c6b848c2fdd5af362383bf0e8ef072900a0575" +version = "1.58.10" +source = "git+https://github.com/SmooAI/smooth-operator.git?rev=9b30ed7b#9b30ed7b96817f6a721584c11eee5c5b515ed4d0" dependencies = [ "anyhow", "async-trait", @@ -5362,9 +5362,9 @@ dependencies = [ [[package]] name = "smooai-smooth-operator-core" -version = "1.7.10" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c3d3d9c931b50ce6f9c262e1d20e82911f5a1001cccd830f5756dd259384f76" +checksum = "457200125a75ce4d1224524ee4021ae3d49ad004a173e49b82712c53a52a3c60" dependencies = [ "anyhow", "async-trait", @@ -5390,8 +5390,8 @@ dependencies = [ [[package]] name = "smooai-smooth-operator-ingestion" -version = "1.44.1" -source = "git+https://github.com/SmooAI/smooth-operator.git?rev=b6c6b84#b6c6b848c2fdd5af362383bf0e8ef072900a0575" +version = "1.58.10" +source = "git+https://github.com/SmooAI/smooth-operator.git?rev=9b30ed7b#9b30ed7b96817f6a721584c11eee5c5b515ed4d0" dependencies = [ "anyhow", "async-trait", @@ -5411,8 +5411,8 @@ dependencies = [ [[package]] name = "smooai-smooth-operator-server" -version = "1.44.1" -source = "git+https://github.com/SmooAI/smooth-operator.git?rev=b6c6b84#b6c6b848c2fdd5af362383bf0e8ef072900a0575" +version = "1.58.10" +source = "git+https://github.com/SmooAI/smooth-operator.git?rev=9b30ed7b#9b30ed7b96817f6a721584c11eee5c5b515ed4d0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 174fc9aa3..c97b5340d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -218,9 +218,9 @@ prost-types = "0.13" # released rather than pinning a side branch — the branch takes the core version # hostage with it. The `th-daemon-memory-seam` and `multimodal-content-model` # branches no longer anchor anything. -smooth-operator = { version = "1.7.10", package = "smooai-smooth-operator-core" } -smooth-operator-server = { git = "https://github.com/SmooAI/smooth-operator.git", rev = "b6c6b84", package = "smooai-smooth-operator-server", default-features = false } -smooth-operator-svc = { git = "https://github.com/SmooAI/smooth-operator.git", rev = "b6c6b84", package = "smooai-smooth-operator", default-features = false } +smooth-operator = { version = "=1.10.0", package = "smooai-smooth-operator-core" } +smooth-operator-server = { git = "https://github.com/SmooAI/smooth-operator.git", rev = "9b30ed7b", package = "smooai-smooth-operator-server", default-features = false } +smooth-operator-svc = { git = "https://github.com/SmooAI/smooth-operator.git", rev = "9b30ed7b", package = "smooai-smooth-operator", default-features = false } # smooth-owned coding-harness extensions to the generic engine (re-homed from # the engine when it went generic at 0.14.0). See crates/smooth-cast. smooth-cast = { version = "0.41.4", path = "crates/smooth-cast", package = "smooai-smooth-cast" } diff --git a/crates/smooth-cast/src/coding_workflow.rs b/crates/smooth-cast/src/coding_workflow.rs index b99703a75..7b3cfbb5a 100644 --- a/crates/smooth-cast/src/coding_workflow.rs +++ b/crates/smooth-cast/src/coding_workflow.rs @@ -464,6 +464,13 @@ pub async fn run_coding_workflow(cfg: CodingWorkflowConfig) -> anyhow::Result { + const m: HistoryMessage = { + direction: 'inbound', + content: { + items: [ + { type: 'text', text: 'look at this' }, + { type: 'image', url: 'data:image/png;base64,AAAA' }, + ], + }, + }; + const imgs = historyImages(m); + assert.equal(imgs.length, 1); + assert.equal(imgs[0].mime, 'image/png'); + assert.equal(imgs[0].dataUrl, 'data:image/png;base64,AAAA'); + assert.ok(imgs[0].mime.startsWith('image/')); // the App.tsx render gate +}); + +test('historyImages defaults https image URLs to image/* so they still render', () => { + const m: HistoryMessage = { + content: { items: [{ type: 'image', url: 'https://cdn/x.png' }] }, + }; + const imgs = historyImages(m); + assert.equal(imgs.length, 1); + assert.equal(imgs[0].mime, 'image/*'); + assert.ok(imgs[0].mime.startsWith('image/')); +}); + +test('historyImages is empty for text-only turns and old daemons (no image items)', () => { + assert.deepEqual(historyImages({ content: { items: [{ type: 'text', text: 'hi' }] } }), []); + assert.deepEqual(historyImages({ content: 'plain string' }), []); + assert.deepEqual(historyImages({ text: 'alias' }), []); +}); diff --git a/crates/smooth-web/web/src/history.ts b/crates/smooth-web/web/src/history.ts new file mode 100644 index 000000000..346d0debb --- /dev/null +++ b/crates/smooth-web/web/src/history.ts @@ -0,0 +1,61 @@ +// Pure parsing of server conversation history into the shapes the UI renders. +// Kept in its own module (no runtime imports) so it's unit-testable under node +// — operator.ts uses Vite-only extensionless imports and can't be loaded by the +// test runner. th-1fca98. + +import type { Attachment } from './operator'; + +/** A raw history message from `get_conversation_messages`. The server returns + * the stored domain `Message`: `direction` ('inbound' = user, 'outbound' = agent) + * + `content: { items: [{ type:'text', text } | { type:'image', url }] }`. + * Fallbacks (`role`, string content, `text`) tolerate other shapes. Past tool + * calls are NOT reconstructed; only the text + images of each turn are rendered. */ +export interface HistoryContentItem { + type?: string; + text?: string; + /** Set on `type:'image'` items — a `data:`/`https` image URL the engine now + * persists on the user turn (th-1fca98), so history re-renders images another + * client attached. */ + url?: string; +} + +export interface HistoryMessage { + direction?: string; + content?: { items?: HistoryContentItem[] } | string; + role?: string; + text?: string; + /** ISO-8601 send time — used to render history oldest-first (the server + * returns newest-first). */ + createdAt?: string; +} + +/** Flatten a history message's content to text (real shape = content.items[] of + * text parts; tolerate a bare string or a `text` alias). */ +export function historyText(m: HistoryMessage): string { + const c = m.content; + if (c && typeof c === 'object' && Array.isArray(c.items)) { + return c.items + .filter((i) => (i.type ?? 'text') === 'text') + .map((i) => i.text ?? '') + .join(''); + } + if (typeof c === 'string') return c; + return m.text ?? ''; +} + +/** The images persisted on a history message (`content.items[]` of `type:'image'`), + * as renderable attachments. Empty for text-only turns and old daemons that never + * stored images. mime comes from a `data:` URL prefix; an `https` URL defaults to + * `image/*` (these items are always images). th-1fca98. */ +export function historyImages(m: HistoryMessage): Attachment[] { + const c = m.content; + if (!c || typeof c !== 'object' || !Array.isArray(c.items)) return []; + const out: Attachment[] = []; + for (const i of c.items) { + if (i.type === 'image' && i.url) { + const mime = i.url.match(/^data:([^;,]+)/)?.[1] ?? 'image/*'; + out.push({ name: 'image', mime, dataUrl: i.url }); + } + } + return out; +} diff --git a/crates/smooth-web/web/src/operator.ts b/crates/smooth-web/web/src/operator.ts index c8004b158..f9905f1b6 100644 --- a/crates/smooth-web/web/src/operator.ts +++ b/crates/smooth-web/web/src/operator.ts @@ -8,6 +8,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { historyImages, historyText, type HistoryMessage } from './history'; import { DEFAULT_MODE_ID, modeById, type ModelCosts, type SmoothMode } from './modes'; import { normalizeTodos, type TodoItem } from './todos'; @@ -90,39 +91,6 @@ export interface ConversationSummary { messageCount: number; } -/** A raw history message from `get_conversation_messages`. The server returns - * the stored domain `Message`: `direction` ('inbound' = user, 'outbound' = agent) - * + `content: { items: [{ type:'text', text }] }`. Fallbacks (`role`, string - * content, `text`) tolerate other shapes. Past tool calls are NOT reconstructed; - * only the text of each turn is rendered. */ -interface HistoryContentItem { - type?: string; - text?: string; -} -interface HistoryMessage { - direction?: string; - content?: { items?: HistoryContentItem[] } | string; - role?: string; - text?: string; - /** ISO-8601 send time — used to render history oldest-first (the server - * returns newest-first). */ - createdAt?: string; -} - -/** Flatten a history message's content to text (real shape = content.items[] of - * text parts; tolerate a bare string or a `text` alias). */ -function historyText(m: HistoryMessage): string { - const c = m.content; - if (c && typeof c === 'object' && Array.isArray(c.items)) { - return c.items - .filter((i) => (i.type ?? 'text') === 'text') - .map((i) => i.text ?? '') - .join(''); - } - if (typeof c === 'string') return c; - return m.text ?? ''; -} - /** Render server history into our ChatMessage model: `inbound` = user turn, * `outbound`/other = assistant (or system). Assistant text becomes one text block. * The server returns messages newest-first, so we sort ascending by `createdAt` @@ -133,6 +101,7 @@ function renderHistory(raw: HistoryMessage[]): ChatMessage[] { const content = historyText(m); const isUser = m.direction === 'inbound' || m.role === 'user'; const role: ChatMessage['role'] = isUser ? 'user' : m.role === 'system' ? 'system' : 'assistant'; + const attachments = isUser ? historyImages(m) : []; return { id: nextId('h'), role, @@ -141,6 +110,7 @@ function renderHistory(raw: HistoryMessage[]): ChatMessage[] { tools: [], blocks: role === 'assistant' ? [{ kind: 'text', text: content }] : [], streaming: false, + attachments: attachments.length ? attachments : undefined, }; }); }