[Feat] Add bounded computer use to Linux sandboxes

apps/docs/desktop-streaming.mdx

@@ -81,6 +81,40 @@ cookies, and logins are shared.
 
 Tasks without a Shared Desktop keep using a private headless browser.
 
+## Opt in to bounded computer use
+
+Environment owners can expose the Cua Driver MCP to the Roomote agent for an
+initial browser-only computer-use preview. Add `computer_use` to the
+environment definition and list every HTTP(S) origin that the agent may open:
+
+```yaml
+computer_use:
+  provider: cua-driver
+  browser_origins:
+    - http://127.0.0.1:3000
+    - https://staging.example.com
+```
+
+Each entry must be an origin only, with no path, query, or fragment. Computer
+use is not enabled unless this section is present. The worker starts Cua Driver
+in bounded mode with a short-lived generated manifest, typed browser tools,
+and the configured origin list. It attaches to Roomote's existing shared
+browser profile; it cannot launch other applications or a separate browser.
+Generic desktop capture and generic native mouse or keyboard tools stay
+disabled.
+
+Cua Driver uses the same X11 display as Shared Desktop. When a person clicks,
+scrolls, types, or pastes through Shared Desktop, read-only Cua observations
+remain available but Cua actions are refused until control returns to the
+agent. Actions also fail closed when the handoff state cannot be read. Stopping
+the task closes the MCP process through the normal agent-runtime cancellation
+path.
+
+This preview does not send a provider credential into the sandbox and does not
+connect Roomote's Jev judgment calls to the computer-use loop. It uses Cua
+Driver's released semantic accessibility and Chromium DOM routes. Cua's
+optional visual-perception extension and its model artifacts are not installed.
+
 ## Verify performance
 
 Open **Shared Desktop**. (A **Start remote desktop** button appears only if
@@ -123,3 +157,9 @@ picture arrives after the encode and network round trip.
 - browser playback uses fragmented MP4 over HTTP, not adaptive bitrate or
   WebRTC congestion control
 - every viewer sees the same desktop; there is one screen and one pointer
+- the computer-use preview is Linux X11 and browser-only; it does not control a
+  person's computer, enable generic native desktop input, or support macOS,
+  Windows, or Wayland sandboxes
+- configured origins bound navigation and typed browser actions, but the
+  capability manifest is not a sandbox around processes already running in the
+  task workspace

apps/worker/Dockerfile

@@ -9,6 +9,7 @@ FROM ubuntu:24.04 AS runtime
 LABEL org.opencontainers.image.source=https://github.com/RooCodeInc/Roomote
 
 ARG AGENT_BROWSER_VERSION=0.37.0
+ARG CUA_DRIVER_VERSION=0.28.2
 ARG OPENCODE_CLI_VERSION=1.18.10
 ARG FFMPEG_INSTALLER_VERSION=1.1.0
 # @ffprobe-installer (not ffprobe-static, which ships no linux/arm64 binary
@@ -69,9 +70,11 @@ RUN apt-get update \
     libx11-xcb1 \
     libxcomposite1 \
     libxdamage1 \
+    libxi6 \
     libxkbcommon0 \
     libxrandr2 \
     libxss1 \
+    at-spi2-core \
   && ln -sf "$(command -v python3)" /usr/local/bin/python \
   && docker compose version \
   && rm -rf /var/lib/apt/lists/*
@@ -211,6 +214,24 @@ ENV SHELL=/bin/bash
 RUN bash /tmp/install-browser-agent.sh \
   && sudo rm -rf /var/lib/apt/lists/*
 
+# Cua Driver is dormant unless an environment explicitly enables bounded
+# computer use. Pin the exact release and verify the platform archive before
+# exposing the binary to task MCP configuration.
+RUN ARCH="$(dpkg --print-architecture)" \
+  && case "$ARCH" in \
+    amd64) CUA_ARCH="x86_64"; CUA_SHA256="a1d99fd04bb4927ef5ffdbe60eb91ed8b51a2bab60e10fc604a75bd59ce69c3e" ;; \
+    arm64) CUA_ARCH="arm64"; CUA_SHA256="55e8a32839a4ac369a773df4dac87b345bd4567779221ade4a5e39223a45a2e8" ;; \
+    *) echo "Unsupported architecture for Cua Driver: $ARCH" >&2; exit 1 ;; \
+  esac \
+  && CUA_ARCHIVE="cua-driver-rs-${CUA_DRIVER_VERSION}-linux-${CUA_ARCH}-binary.tar.gz" \
+  && curl -fsSL "https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_VERSION}/${CUA_ARCHIVE}" -o "/tmp/${CUA_ARCHIVE}" \
+  && printf '%s  %s\n' "$CUA_SHA256" "/tmp/${CUA_ARCHIVE}" | sha256sum -c - \
+  && mkdir -p /tmp/cua-driver \
+  && tar -xzf "/tmp/${CUA_ARCHIVE}" -C /tmp/cua-driver \
+  && sudo install -m 0755 /tmp/cua-driver/cua-driver /usr/local/bin/cua-driver \
+  && test "$(cua-driver --version)" = "cua-driver ${CUA_DRIVER_VERSION}" \
+  && rm -rf /tmp/cua-driver "/tmp/${CUA_ARCHIVE}"
+
 RUN sudo mkdir -p /opt/ffmpeg/bin \
   && sudo chown -R roomote:roomote /opt/ffmpeg \
   && npm install -g \

apps/worker/src/commands/setup/setup-mcps.ts

@@ -37,6 +37,16 @@ const roomoteMcpPath = process.argv[1]
       'worker/dist/mcp/roomote-mcp-server/index.js',
     );
 
+const cuaDriverProxyPath = process.argv[1]
+  ? path.join(
+      path.dirname(path.resolve(process.argv[1])),
+      'mcp/cua-driver-proxy/index.js',
+    )
+  : path.join(
+      process.env.HOME || '/home/roomote',
+      'worker/dist/mcp/cua-driver-proxy/index.js',
+    );
+
 /**
  * Built-in MCP servers enabled for all cloud agents.
  * Add entries here to make them available to the active OpenCode runtime.
@@ -409,6 +419,27 @@ export function resolveBuiltInMcpServers(
     }
   }
 
+  if (
+    taskEnv?.ROOMOTE_CUA_DRIVER_BINARY &&
+    taskEnv.ROOMOTE_CUA_DRIVER_MANIFEST_PATH &&
+    taskEnv.ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL
+  ) {
+    resolvedMcps['cua-driver'] = {
+      type: 'stdio',
+      command: 'node',
+      args: [cuaDriverProxyPath],
+      env: {
+        ...stdioEnvExtras,
+        DISPLAY: taskEnv.DISPLAY ?? '',
+        ROOMOTE_CUA_DRIVER_BINARY: taskEnv.ROOMOTE_CUA_DRIVER_BINARY,
+        ROOMOTE_CUA_DRIVER_MANIFEST_PATH:
+          taskEnv.ROOMOTE_CUA_DRIVER_MANIFEST_PATH,
+        ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL:
+          taskEnv.ROOMOTE_CUA_DRIVER_HUMAN_CONTROL_URL,
+      },
+    };
+  }
+
   // Add integration-provided MCP servers.
   if (integrations?.userMcpServers) {
     for (const [name, config] of Object.entries(integrations.userMcpServers)) {

apps/worker/src/commands/setup/workspace/services.ts

@@ -9,6 +9,7 @@ import {
 import type { StartupLogger } from '../../../logging';
 import {
   type ServiceContext,
+  configureCuaDriver,
   ServiceManager,
   startPortProxies,
   startSharedDesktop,
@@ -143,6 +144,7 @@ async function startEnvironmentServices({
   logger: StartupLogger;
 }): Promise<ServiceInfo[]> {
   const services: ServiceInfo[] = [];
+  let sharedDesktopStarted = false;
 
   if (
     environmentConfig &&
@@ -151,20 +153,50 @@ async function startEnvironmentServices({
     // Shared Desktop is optional: a failure here must not prevent the port
     // proxies below from starting, or Live Preview breaks alongside it.
     try {
-      await timedStep(logger, 'start shared desktop', () =>
-        startSharedDesktop({
-          cwd: workspaceRoot,
+      sharedDesktopStarted = await timedStep(
+        logger,
+        'start shared desktop',
+        () =>
+          startSharedDesktop({
+            cwd: workspaceRoot,
+            env: envVars,
+            allowedControlOrigin: serviceContext.appOrigin,
+          }),
+      );
+    } catch (error) {
+      logger.userLog.warn(
+        `Shared Desktop is unavailable for this task: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+    }
+  }
+
+  const computerUseConfig = environmentConfig?.computer_use;
+  if (computerUseConfig && sharedDesktopStarted) {
+    try {
+      const configured = await timedStep(logger, 'configure computer use', () =>
+        configureCuaDriver({
+          config: computerUseConfig,
           env: envVars,
-          allowedControlOrigin: serviceContext.appOrigin,
         }),
       );
+      if (!configured) {
+        logger.userLog.warn(
+          'Computer use is unavailable because this worker image does not include Cua Driver',
+        );
+      }
     } catch (error) {
       logger.userLog.warn(
-        `Shared Desktop is unavailable for this task: ${
+        `Computer use is unavailable for this task: ${
           error instanceof Error ? error.message : String(error)
         }`,
       );
     }
+  } else if (computerUseConfig) {
+    logger.userLog.warn(
+      'Computer use is unavailable because the Shared Desktop did not start',
+    );
   }
 
   // Start port proxies if proxyPorts are configured.