From c14904582390fa7f41d4225c8ff130e6b23ca6c0 Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Sat, 19 Sep 2026 02:35:37 +0000 Subject: [PATCH 1/2] feat: add bounded sandbox computer use --- apps/docs/desktop-streaming.mdx | 40 ++++++++ apps/worker/Dockerfile | 21 +++++ .../setup/__tests__/setup-mcps.test.ts | 28 ++++++ apps/worker/src/commands/setup/setup-mcps.ts | 31 +++++++ .../src/commands/setup/workspace/services.ts | 42 ++++++++- .../src/mcp/cua-driver-proxy/guard.test.ts | 76 +++++++++++++++ apps/worker/src/mcp/cua-driver-proxy/guard.ts | 82 ++++++++++++++++ apps/worker/src/mcp/cua-driver-proxy/index.ts | 89 ++++++++++++++++++ .../__tests__/cua-driver-service.test.ts | 85 +++++++++++++++++ .../worker/src/services/cua-driver-service.ts | 93 +++++++++++++++++++ apps/worker/src/services/index.ts | 1 + apps/worker/tsup.config.ts | 1 + .../src/__tests__/command-schema.test.ts | 37 ++++++++ packages/types/src/environment-config.ts | 30 ++++++ 14 files changed, 651 insertions(+), 5 deletions(-) create mode 100644 apps/worker/src/mcp/cua-driver-proxy/guard.test.ts create mode 100644 apps/worker/src/mcp/cua-driver-proxy/guard.ts create mode 100644 apps/worker/src/mcp/cua-driver-proxy/index.ts create mode 100644 apps/worker/src/services/__tests__/cua-driver-service.test.ts create mode 100644 apps/worker/src/services/cua-driver-service.ts diff --git a/apps/docs/desktop-streaming.mdx b/apps/docs/desktop-streaming.mdx index f057f2df13..c8066d78dd 100644 --- a/apps/docs/desktop-streaming.mdx +++ b/apps/docs/desktop-streaming.mdx @@ -81,6 +81,40 @@ cookies, and logins are shared. Tasks without a Shared Desktop keep using a private headless browser. +## Opt in to bounded computer use + +Environment owners can expose the Cua Driver MCP to the Roomote agent for an +initial browser-only computer-use preview. Add `computer_use` to the +environment definition and list every HTTP(S) origin that the agent may open: + +```yaml +computer_use: + provider: cua-driver + browser_origins: + - http://127.0.0.1:3000 + - https://staging.example.com +``` + +Each entry must be an origin only, with no path, query, or fragment. Computer +use is not enabled unless this section is present. The worker starts Cua Driver +in bounded mode with a short-lived generated manifest, typed browser tools, +and the configured origin list. It attaches to Roomote's existing shared +browser profile; it cannot launch other applications or a separate browser. +Generic desktop capture and generic native mouse or keyboard tools stay +disabled. + +Cua Driver uses the same X11 display as Shared Desktop. When a person clicks, +scrolls, types, or pastes through Shared Desktop, read-only Cua observations +remain available but Cua actions are refused until control returns to the +agent. Actions also fail closed when the handoff state cannot be read. Stopping +the task closes the MCP process through the normal agent-runtime cancellation +path. + +This preview does not send a provider credential into the sandbox and does not +connect Roomote's Jev judgment calls to the computer-use loop. It uses Cua +Driver's released semantic accessibility and Chromium DOM routes. Cua's +optional visual-perception extension and its model artifacts are not installed. + ## Verify performance Open **Shared Desktop**. (A **Start remote desktop** button appears only if @@ -123,3 +157,9 @@ picture arrives after the encode and network round trip. - browser playback uses fragmented MP4 over HTTP, not adaptive bitrate or WebRTC congestion control - every viewer sees the same desktop; there is one screen and one pointer +- the computer-use preview is Linux X11 and browser-only; it does not control a + person's computer, enable generic native desktop input, or support macOS, + Windows, or Wayland sandboxes +- configured origins bound navigation and typed browser actions, but the + capability manifest is not a sandbox around processes already running in the + task workspace diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 93e1717971..9a42e63d91 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -9,6 +9,7 @@ FROM ubuntu:24.04 AS runtime LABEL org.opencontainers.image.source=https://github.com/RooCodeInc/Roomote ARG AGENT_BROWSER_VERSION=0.37.0 +ARG CUA_DRIVER_VERSION=0.28.2 ARG OPENCODE_CLI_VERSION=1.18.10 ARG FFMPEG_INSTALLER_VERSION=1.1.0 # @ffprobe-installer (not ffprobe-static, which ships no linux/arm64 binary @@ -69,9 +70,11 @@ RUN apt-get update \ libx11-xcb1 \ libxcomposite1 \ libxdamage1 \ + libxi6 \ libxkbcommon0 \ libxrandr2 \ libxss1 \ + at-spi2-core \ && ln -sf "$(command -v python3)" /usr/local/bin/python \ && docker compose version \ && rm -rf /var/lib/apt/lists/* @@ -211,6 +214,24 @@ ENV SHELL=/bin/bash RUN bash /tmp/install-browser-agent.sh \ && sudo rm -rf /var/lib/apt/lists/* +# Cua Driver is dormant unless an environment explicitly enables bounded +# computer use. Pin the exact release and verify the platform archive before +# exposing the binary to task MCP configuration. +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in \ + amd64) CUA_ARCH="x86_64"; CUA_SHA256="a1d99fd04bb4927ef5ffdbe60eb91ed8b51a2bab60e10fc604a75bd59ce69c3e" ;; \ + arm64) CUA_ARCH="arm64"; CUA_SHA256="55e8a32839a4ac369a773df4dac87b345bd4567779221ade4a5e39223a45a2e8" ;; \ + *) echo "Unsupported architecture for Cua Driver: $ARCH" >&2; exit 1 ;; \ + esac \ + && CUA_ARCHIVE="cua-driver-rs-${CUA_DRIVER_VERSION}-linux-${CUA_ARCH}-binary.tar.gz" \ + && curl -fsSL "https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_VERSION}/${CUA_ARCHIVE}" -o "/tmp/${CUA_ARCHIVE}" \ + && printf '%s %s\n' "$CUA_SHA256" "/tmp/${CUA_ARCHIVE}" | sha256sum -c - \ + && mkdir -p /tmp/cua-driver \ + && tar -xzf "/tmp/${CUA_ARCHIVE}" -C /tmp/cua-driver \ + && sudo install -m 0755 /tmp/cua-driver/cua-driver /usr/local/bin/cua-driver \ + && test "$(cua-driver --version)" = "cua-driver ${CUA_DRIVER_VERSION}" \ + && rm -rf /tmp/cua-driver "/tmp/${CUA_ARCHIVE}" + RUN sudo mkdir -p /opt/ffmpeg/bin \ && sudo chown -R roomote:roomote /opt/ffmpeg \ && npm install -g \ diff --git a/apps/worker/src/commands/setup/__tests__/setup-mcps.test.ts b/apps/worker/src/commands/setup/__tests__/setup-mcps.test.ts index bfb32299b2..fc5033eee2 100644 --- a/apps/worker/src/commands/setup/__tests__/setup-mcps.test.ts +++ b/apps/worker/src/commands/setup/__tests__/setup-mcps.test.ts @@ -73,6 +73,34 @@ describe('resolveBuiltInMcpServers', () => { expect(Object.keys(BUILT_IN_MCPS).sort()).toEqual(expectedBuiltInMcpNames); }); + it('mounts the guarded Cua Driver MCP only after the desktop configured it', () => { + const disabled = resolveBuiltInMcpServers({ DISPLAY: ':99' }); + expect(disabled).not.toHaveProperty('cua-driver'); + + const enabled = resolveBuiltInMcpServers({ + DISPLAY: ':99', + ROOMOTE_CUA_DRIVER_BINARY: '/usr/local/bin/cua-driver', + ROOMOTE_CUA_DRIVER_MANIFEST_PATH: + '/home/roomote/.roomote/cua-driver/capabilities.yaml', + ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL: 'http://127.0.0.1:6080/metrics', + }); + + expect(enabled['cua-driver']).toEqual({ + type: 'stdio', + command: 'node', + args: [expect.stringMatching(/mcp\/cua-driver-proxy\/index\.js$/)], + env: { + MISE_DATA_DIR: '/opt/mise', + MISE_CACHE_DIR: '/opt/mise/cache', + DISPLAY: ':99', + ROOMOTE_CUA_DRIVER_BINARY: '/usr/local/bin/cua-driver', + ROOMOTE_CUA_DRIVER_MANIFEST_PATH: + '/home/roomote/.roomote/cua-driver/capabilities.yaml', + ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL: 'http://127.0.0.1:6080/metrics', + }, + }); + }); + it.each([ '/api/mcp/http-integrations', 'https://web.test/api/mcp/http-integrations', diff --git a/apps/worker/src/commands/setup/setup-mcps.ts b/apps/worker/src/commands/setup/setup-mcps.ts index e847defc3e..e380a1cd10 100644 --- a/apps/worker/src/commands/setup/setup-mcps.ts +++ b/apps/worker/src/commands/setup/setup-mcps.ts @@ -37,6 +37,16 @@ const roomoteMcpPath = process.argv[1] 'worker/dist/mcp/roomote-mcp-server/index.js', ); +const cuaDriverProxyPath = process.argv[1] + ? path.join( + path.dirname(path.resolve(process.argv[1])), + 'mcp/cua-driver-proxy/index.js', + ) + : path.join( + process.env.HOME || '/home/roomote', + 'worker/dist/mcp/cua-driver-proxy/index.js', + ); + /** * Built-in MCP servers enabled for all cloud agents. * Add entries here to make them available to the active OpenCode runtime. @@ -409,6 +419,27 @@ export function resolveBuiltInMcpServers( } } + if ( + taskEnv?.ROOMOTE_CUA_DRIVER_BINARY && + taskEnv.ROOMOTE_CUA_DRIVER_MANIFEST_PATH && + taskEnv.ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL + ) { + resolvedMcps['cua-driver'] = { + type: 'stdio', + command: 'node', + args: [cuaDriverProxyPath], + env: { + ...stdioEnvExtras, + DISPLAY: taskEnv.DISPLAY ?? '', + ROOMOTE_CUA_DRIVER_BINARY: taskEnv.ROOMOTE_CUA_DRIVER_BINARY, + ROOMOTE_CUA_DRIVER_MANIFEST_PATH: + taskEnv.ROOMOTE_CUA_DRIVER_MANIFEST_PATH, + ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL: + taskEnv.ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL, + }, + }; + } + // Add integration-provided MCP servers. if (integrations?.userMcpServers) { for (const [name, config] of Object.entries(integrations.userMcpServers)) { diff --git a/apps/worker/src/commands/setup/workspace/services.ts b/apps/worker/src/commands/setup/workspace/services.ts index 547e50d773..229df711b3 100644 --- a/apps/worker/src/commands/setup/workspace/services.ts +++ b/apps/worker/src/commands/setup/workspace/services.ts @@ -9,6 +9,7 @@ import { import type { StartupLogger } from '../../../logging'; import { type ServiceContext, + configureCuaDriver, ServiceManager, startPortProxies, startSharedDesktop, @@ -143,6 +144,7 @@ async function startEnvironmentServices({ logger: StartupLogger; }): Promise { const services: ServiceInfo[] = []; + let sharedDesktopStarted = false; if ( environmentConfig && @@ -151,20 +153,50 @@ async function startEnvironmentServices({ // Shared Desktop is optional: a failure here must not prevent the port // proxies below from starting, or Live Preview breaks alongside it. try { - await timedStep(logger, 'start shared desktop', () => - startSharedDesktop({ - cwd: workspaceRoot, + sharedDesktopStarted = await timedStep( + logger, + 'start shared desktop', + () => + startSharedDesktop({ + cwd: workspaceRoot, + env: envVars, + allowedControlOrigin: serviceContext.appOrigin, + }), + ); + } catch (error) { + logger.userLog.warn( + `Shared Desktop is unavailable for this task: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + } + + const computerUseConfig = environmentConfig?.computer_use; + if (computerUseConfig && sharedDesktopStarted) { + try { + const configured = await timedStep(logger, 'configure computer use', () => + configureCuaDriver({ + config: computerUseConfig, env: envVars, - allowedControlOrigin: serviceContext.appOrigin, }), ); + if (!configured) { + logger.userLog.warn( + 'Computer use is unavailable because this worker image does not include Cua Driver', + ); + } } catch (error) { logger.userLog.warn( - `Shared Desktop is unavailable for this task: ${ + `Computer use is unavailable for this task: ${ error instanceof Error ? error.message : String(error) }`, ); } + } else if (computerUseConfig) { + logger.userLog.warn( + 'Computer use is unavailable because the Shared Desktop did not start', + ); } // Start port proxies if proxyPorts are configured. diff --git a/apps/worker/src/mcp/cua-driver-proxy/guard.test.ts b/apps/worker/src/mcp/cua-driver-proxy/guard.test.ts new file mode 100644 index 0000000000..9fb7b1edc6 --- /dev/null +++ b/apps/worker/src/mcp/cua-driver-proxy/guard.test.ts @@ -0,0 +1,76 @@ +import { guardCuaDriverRequest, readHumanControlState } from './guard'; + +describe('readHumanControlState', () => { + it.each([ + [true, 'human'], + [false, 'agent'], + ] as const)('maps human_driving=%s to %s', async (humanDriving, expected) => { + const fetchImpl = vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ human_driving: humanDriving }), + }); + + await expect( + readHumanControlState('http://127.0.0.1:6080/metrics', fetchImpl), + ).resolves.toBe(expected); + }); + + it('fails closed when metrics cannot be read', async () => { + await expect( + readHumanControlState( + 'http://127.0.0.1:6080/metrics', + vi.fn().mockRejectedValue(new Error('offline')), + ), + ).resolves.toBe('unavailable'); + }); +}); + +describe('guardCuaDriverRequest', () => { + const request = (name: string) => + JSON.stringify({ + jsonrpc: '2.0', + id: 7, + method: 'tools/call', + params: { name, arguments: {} }, + }); + + it('keeps observation tools available while a person drives', async () => { + await expect( + guardCuaDriverRequest(request('get_browser_state'), async () => 'human'), + ).resolves.toEqual({ forward: true }); + }); + + it('refuses input tools while a person drives', async () => { + const result = await guardCuaDriverRequest( + request('browser_click'), + async () => 'human', + ); + + expect(result.forward).toBe(false); + if (!result.forward) { + expect(JSON.parse(result.response)).toMatchObject({ + id: 7, + result: { isError: true }, + }); + expect(result.response).toContain('person is using the Shared Desktop'); + } + }); + + it('fails closed for new mutation tools when handoff state is unavailable', async () => { + const result = await guardCuaDriverRequest( + request('future_input_tool'), + async () => 'unavailable', + ); + + expect(result.forward).toBe(false); + if (!result.forward) { + expect(result.response).toContain('handoff state is unavailable'); + } + }); + + it('forwards input after control returns to the agent', async () => { + await expect( + guardCuaDriverRequest(request('browser_type'), async () => 'agent'), + ).resolves.toEqual({ forward: true }); + }); +}); diff --git a/apps/worker/src/mcp/cua-driver-proxy/guard.ts b/apps/worker/src/mcp/cua-driver-proxy/guard.ts new file mode 100644 index 0000000000..b0ee101584 --- /dev/null +++ b/apps/worker/src/mcp/cua-driver-proxy/guard.ts @@ -0,0 +1,82 @@ +const READ_ONLY_CUA_TOOLS = new Set([ + 'check_for_update', + 'clipboard_read', + 'get_browser_state', + 'get_desktop_state', + 'get_session_state', + 'get_window_state', + 'history_query', + 'history_status', + 'list_apps', + 'list_recordings', + 'list_windows', + 'recording_status', + 'verify_state', +]); + +type JsonRpcRequest = { + jsonrpc?: unknown; + id?: unknown; + method?: unknown; + params?: { name?: unknown }; +}; + +export type HumanControlState = 'agent' | 'human' | 'unavailable'; + +export async function readHumanControlState( + metricsUrl: string, + fetchImpl: typeof fetch = fetch, +): Promise { + try { + const response = await fetchImpl(metricsUrl, { + signal: AbortSignal.timeout(2_000), + }); + if (!response.ok) return 'unavailable'; + + const body = (await response.json()) as { human_driving?: unknown }; + if (body.human_driving === true) return 'human'; + if (body.human_driving === false) return 'agent'; + return 'unavailable'; + } catch { + return 'unavailable'; + } +} + +export async function guardCuaDriverRequest( + line: string, + readState: () => Promise, +): Promise<{ forward: true } | { forward: false; response: string }> { + let request: JsonRpcRequest; + try { + request = JSON.parse(line) as JsonRpcRequest; + } catch { + return { forward: true }; + } + + if (request.method !== 'tools/call') return { forward: true }; + + const toolName = request.params?.name; + if (typeof toolName !== 'string' || READ_ONLY_CUA_TOOLS.has(toolName)) { + return { forward: true }; + } + + const state = await readState(); + if (state === 'agent') return { forward: true }; + + const message = + state === 'human' + ? 'A person is using the Shared Desktop, so this computer-use action was not run. Observation tools remain available; retry after control returns to the agent.' + : 'Shared Desktop handoff state is unavailable, so this computer-use action was not run. Observation tools remain available; retry after the desktop service recovers.'; + + return { + forward: false, + response: JSON.stringify({ + jsonrpc: '2.0', + id: request.id ?? null, + result: { + content: [{ type: 'text', text: message }], + isError: true, + }, + }), + }; +} diff --git a/apps/worker/src/mcp/cua-driver-proxy/index.ts b/apps/worker/src/mcp/cua-driver-proxy/index.ts new file mode 100644 index 0000000000..b7a57f42d9 --- /dev/null +++ b/apps/worker/src/mcp/cua-driver-proxy/index.ts @@ -0,0 +1,89 @@ +import { spawn } from 'node:child_process'; +import { createInterface } from 'node:readline'; + +import { guardCuaDriverRequest, readHumanControlState } from './guard'; + +const binary = process.env.ROOMOTE_CUA_DRIVER_BINARY; +const manifestPath = process.env.ROOMOTE_CUA_DRIVER_MANIFEST_PATH; +const humanControlUrl = process.env.ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL; + +if (!binary || !manifestPath || !humanControlUrl) { + console.error('cua-driver proxy: missing required runtime configuration'); + process.exit(1); +} + +const childEnv: Record = { + ...Object.fromEntries( + [ + 'AT_SPI_BUS_ADDRESS', + 'DBUS_SESSION_BUS_ADDRESS', + 'DISPLAY', + 'HOME', + 'LANG', + 'LC_ALL', + 'PATH', + 'XAUTHORITY', + 'XDG_RUNTIME_DIR', + ].flatMap((name) => + process.env[name] ? [[name, process.env[name]!]] : [], + ), + ), + CUA_DRIVER_PERMISSION_MODE: 'bounded', + CUA_DRIVER_CAPABILITY_MANIFEST_FILE: manifestPath, + CUA_DRIVER_CAPABILITY_MANIFEST_APPROVED: '1', +}; + +const driver = spawn(binary, ['mcp'], { + env: childEnv, + stdio: ['pipe', 'pipe', 'pipe'], +}); + +driver.stdout.pipe(process.stdout); +driver.stderr.pipe(process.stderr); + +let inputQueue = Promise.resolve(); +const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); + +input.on('line', (line) => { + inputQueue = inputQueue + .then(async () => { + const guarded = await guardCuaDriverRequest(line, () => + readHumanControlState(humanControlUrl), + ); + if (guarded.forward) { + driver.stdin.write(`${line}\n`); + } else { + process.stdout.write(`${guarded.response}\n`); + } + }) + .catch((error) => { + console.error( + `cua-driver proxy: failed to process request: ${error instanceof Error ? error.message : String(error)}`, + ); + }); +}); + +input.on('close', () => { + void inputQueue.finally(() => driver.stdin.end()); +}); + +let stopping = false; +function stopDriver(signal: NodeJS.Signals): void { + if (stopping) return; + stopping = true; + driver.kill(signal); + const forceKill = setTimeout(() => driver.kill('SIGKILL'), 2_000); + forceKill.unref(); +} + +process.once('SIGINT', () => stopDriver('SIGINT')); +process.once('SIGTERM', () => stopDriver('SIGTERM')); + +driver.once('error', (error) => { + console.error(`cua-driver proxy: failed to start Driver: ${error.message}`); + process.exitCode = 1; +}); + +driver.once('exit', (code, signal) => { + process.exit(code ?? (signal ? 1 : 0)); +}); diff --git a/apps/worker/src/services/__tests__/cua-driver-service.test.ts b/apps/worker/src/services/__tests__/cua-driver-service.test.ts new file mode 100644 index 0000000000..815fc137a0 --- /dev/null +++ b/apps/worker/src/services/__tests__/cua-driver-service.test.ts @@ -0,0 +1,85 @@ +import { access, mkdir, writeFile } from 'node:fs/promises'; + +import { + CUA_DRIVER_MANIFEST_PATH, + buildCuaDriverCapabilityManifest, + configureCuaDriver, +} from '../cua-driver-service'; + +vi.mock('node:fs/promises', () => ({ + access: vi.fn(), + mkdir: vi.fn(), + writeFile: vi.fn(), +})); + +describe('buildCuaDriverCapabilityManifest', () => { + it('allows only typed browser tools and keeps desktop access disabled', () => { + const manifest = buildCuaDriverCapabilityManifest({ + provider: 'cua-driver', + browser_origins: ['http://127.0.0.1:4173'], + }); + + expect(manifest).toContain('browser_navigate'); + expect(manifest).toContain('http://127.0.0.1:4173'); + expect(manifest).toContain('/opt/agent-browser/chrome'); + expect(manifest).toContain('kind: existing_profile'); + expect(manifest).not.toContain('kind: isolated'); + expect(manifest).toContain('display: false'); + expect(manifest).not.toContain('get_desktop_state'); + expect(manifest).not.toContain('type_text'); + }); +}); + +describe('configureCuaDriver', () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(access).mockResolvedValue(undefined); + vi.mocked(mkdir).mockResolvedValue(undefined); + vi.mocked(writeFile).mockResolvedValue(undefined); + }); + + it('writes a private manifest and exports the guarded MCP runtime config', async () => { + const env: Record = { + DISPLAY: ':99', + ROOMOTE_DESKTOP_STREAM_PORT: '6080', + }; + + await expect( + configureCuaDriver({ + config: { + provider: 'cua-driver', + browser_origins: ['http://127.0.0.1:4173'], + }, + env, + }), + ).resolves.toBe(true); + + expect(writeFile).toHaveBeenCalledWith( + CUA_DRIVER_MANIFEST_PATH, + expect.stringContaining('http://127.0.0.1:4173'), + { mode: 0o600 }, + ); + expect(env).toMatchObject({ + ROOMOTE_CUA_DRIVER_BINARY: '/usr/local/bin/cua-driver', + ROOMOTE_CUA_DRIVER_MANIFEST_PATH: CUA_DRIVER_MANIFEST_PATH, + ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL: 'http://127.0.0.1:6080/metrics', + }); + }); + + it('does not enable the MCP without a running Shared Desktop', async () => { + const env: Record = {}; + + await expect( + configureCuaDriver({ + config: { + provider: 'cua-driver', + browser_origins: ['http://127.0.0.1:4173'], + }, + env, + }), + ).resolves.toBe(false); + + expect(writeFile).not.toHaveBeenCalled(); + expect(env).not.toHaveProperty('ROOMOTE_CUA_DRIVER_MANIFEST_PATH'); + }); +}); diff --git a/apps/worker/src/services/cua-driver-service.ts b/apps/worker/src/services/cua-driver-service.ts new file mode 100644 index 0000000000..ca98e83cdb --- /dev/null +++ b/apps/worker/src/services/cua-driver-service.ts @@ -0,0 +1,93 @@ +import YAML from 'yaml'; +import { access, mkdir, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +import type { EnvironmentComputerUse } from '@roomote/types'; + +const CUA_DRIVER_BINARY = '/usr/local/bin/cua-driver'; +const SHARED_BROWSER_BINARY = '/opt/agent-browser/chrome'; +const CUA_DRIVER_STATE_DIR = '/home/roomote/.roomote/cua-driver'; +export const CUA_DRIVER_MANIFEST_PATH = join( + CUA_DRIVER_STATE_DIR, + 'capabilities.yaml', +); + +const CUA_DRIVER_BROWSER_TOOLS = [ + 'start_session', + 'end_session', + 'list_apps', + 'list_windows', + 'browser_prepare', + 'get_browser_state', + 'browser_navigate', + 'browser_click', + 'browser_type', + 'browser_download', +] as const; + +export function buildCuaDriverCapabilityManifest( + config: EnvironmentComputerUse, +): string { + return YAML.stringify({ + version: 3, + expires_after: '8h', + idle_timeout: '30m', + allow: { tools: CUA_DRIVER_BROWSER_TOOLS }, + resources: { + apps: [ + { + executable: SHARED_BROWSER_BINARY, + launch: false, + windows: 'all', + }, + ], + browser: { + profiles: [{ kind: 'existing_profile' }], + origins: config.browser_origins, + }, + desktop: { display: false }, + }, + }); +} + +/** + * Enables the task-local MCP only when the pinned Driver binary and the + * Shared Desktop it operates are both available. The MCP process itself is + * owned by the harness, so normal task cancellation tears it down. + */ +export async function configureCuaDriver(params: { + config: EnvironmentComputerUse; + env: Record; +}): Promise { + try { + await access(CUA_DRIVER_BINARY); + } catch { + console.warn( + 'Cua Driver is unavailable because this worker image predates computer use support', + ); + return false; + } + + if (!params.env.DISPLAY || !params.env.ROOMOTE_DESKTOP_STREAM_PORT) { + console.warn( + 'Cua Driver is unavailable because the Shared Desktop did not start', + ); + return false; + } + + await mkdir(dirname(CUA_DRIVER_MANIFEST_PATH), { + recursive: true, + mode: 0o700, + }); + await writeFile( + CUA_DRIVER_MANIFEST_PATH, + buildCuaDriverCapabilityManifest(params.config), + { mode: 0o600 }, + ); + + params.env.ROOMOTE_CUA_DRIVER_MANIFEST_PATH = CUA_DRIVER_MANIFEST_PATH; + params.env.ROOMOTE_CUA_DRIVER_BINARY = CUA_DRIVER_BINARY; + params.env.ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL = `http://127.0.0.1:${params.env.ROOMOTE_DESKTOP_STREAM_PORT}/metrics`; + + return true; +} diff --git a/apps/worker/src/services/index.ts b/apps/worker/src/services/index.ts index 7a90217a81..9bbc217e3a 100644 --- a/apps/worker/src/services/index.ts +++ b/apps/worker/src/services/index.ts @@ -2,3 +2,4 @@ export type { ServiceContext } from './types'; export { ServiceManager } from './service-manager'; export { startPortProxies } from './port-proxy-service'; export { startSharedDesktop } from './shared-desktop-service'; +export { configureCuaDriver } from './cua-driver-service'; diff --git a/apps/worker/tsup.config.ts b/apps/worker/tsup.config.ts index eea92f1702..9f76dc6f06 100644 --- a/apps/worker/tsup.config.ts +++ b/apps/worker/tsup.config.ts @@ -13,6 +13,7 @@ const jsdomSyncWorkerEntry = join( export default defineConfig({ entry: { worker: 'scripts/worker.ts', + 'mcp/cua-driver-proxy/index': 'src/mcp/cua-driver-proxy/index.ts', 'mcp/roomote-mcp-server/index': 'src/mcp/roomote-mcp-server/index.ts', // JSDOM resolves and launches this helper at runtime even when consumers // never make synchronous XHR requests. Bundle it beside the MCP server so diff --git a/packages/types/src/__tests__/command-schema.test.ts b/packages/types/src/__tests__/command-schema.test.ts index dc6d366083..b6534165b6 100644 --- a/packages/types/src/__tests__/command-schema.test.ts +++ b/packages/types/src/__tests__/command-schema.test.ts @@ -486,6 +486,43 @@ describe('environmentConfigSchema', () => { }); }); + it('accepts opt-in origin-bounded Cua Driver computer use', () => { + expect( + environmentConfigSchema.parse({ + name: 'Computer-use workspace', + computer_use: { + provider: 'cua-driver', + browser_origins: ['http://127.0.0.1:3000', 'https://example.test'], + }, + }).computer_use, + ).toEqual({ + provider: 'cua-driver', + browser_origins: ['http://127.0.0.1:3000', 'https://example.test'], + }); + }); + + const unsafeComputerUseOrigins: string[][] = [ + [], + ['file:///tmp/fixture.html'], + ['https://example.test/path'], + ['https://example.test?query=value'], + ]; + + it.each(unsafeComputerUseOrigins)( + 'rejects an unsafe computer-use origin list: %j', + (browserOrigins) => { + expect( + environmentConfigSchema.safeParse({ + name: 'Computer-use workspace', + computer_use: { + provider: 'cua-driver', + browser_origins: browserOrigins, + }, + }).success, + ).toBe(false); + }, + ); + it.each([ { label: 'top-level env', diff --git a/packages/types/src/environment-config.ts b/packages/types/src/environment-config.ts index c4feeff89e..9c33b6ef3d 100644 --- a/packages/types/src/environment-config.ts +++ b/packages/types/src/environment-config.ts @@ -616,6 +616,30 @@ const environmentInitialUrlSchema = z.union([ z.string().url(), ]); +const computerUseBrowserOriginSchema = z + .string() + .url() + .refine((value) => { + try { + const url = new URL(value); + return ( + (url.protocol === 'http:' || url.protocol === 'https:') && + url.origin === value + ); + } catch { + return false; + } + }, 'Must be an HTTP(S) origin without a path, query, or fragment'); + +export const environmentComputerUseSchema = z.object({ + provider: z.literal('cua-driver'), + browser_origins: z.array(computerUseBrowserOriginSchema).min(1).max(20), +}); + +export type EnvironmentComputerUse = z.infer< + typeof environmentComputerUseSchema +>; + function validatePortLimits(ports: NamedPort[]): boolean { const proxiedPorts = ports.filter((port) => port.proxied !== false); const nonProxiedPorts = ports.filter((port) => port.proxied === false); @@ -730,6 +754,12 @@ export const environmentConfigSchema = z * their repositories have been prepared. */ docker_projects: z.array(dockerProjectSchema).optional(), + /** + * Opt-in, origin-bounded computer use inside the sandbox's Linux desktop. + * The worker exposes a Cua Driver MCP only after the Shared Desktop is + * healthy, and keeps generic desktop capture/input disabled. + */ + computer_use: environmentComputerUseSchema.optional(), /** * Optional sandbox OIDC targets for this environment. * Tokens are minted by Roomote, written into the sandbox filesystem, and From fe771e71778065df57fc17d5d76758ef6795624f Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Sat, 19 Sep 2026 02:37:24 +0000 Subject: [PATCH 2/2] fix: keep Cua handoff state internal --- apps/worker/src/mcp/cua-driver-proxy/guard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/worker/src/mcp/cua-driver-proxy/guard.ts b/apps/worker/src/mcp/cua-driver-proxy/guard.ts index b0ee101584..449e9b5191 100644 --- a/apps/worker/src/mcp/cua-driver-proxy/guard.ts +++ b/apps/worker/src/mcp/cua-driver-proxy/guard.ts @@ -21,7 +21,7 @@ type JsonRpcRequest = { params?: { name?: unknown }; }; -export type HumanControlState = 'agent' | 'human' | 'unavailable'; +type HumanControlState = 'agent' | 'human' | 'unavailable'; export async function readHumanControlState( metricsUrl: string,