test(temporal): Proved any-worker resume across two machines, not two processes.

packages/temporal/README.md

@@ -350,4 +350,7 @@ an outcome query or a person to decide what happened.
 `scripts/detached-session-check.sh` runs the client story against a Temporal dev server, one
 standalone worker, two serve processes and one shared store: a turn started on serve A survives A
 being killed mid-tool, serve B replays it, and `start`, `running`, `watch` and `schedule` work from
-a cold client. It needs an OpenAI key and does not run in CI.
+a cold client. `scripts/cross-host-check.sh` runs the worktree story in containers
+(`docker/compose.yml`): a session writes a file on worker A, A's host is killed, and worker B, whose
+project volume is empty, continues the session and reads the file back. Both need an OpenAI key,
+and neither runs in CI.

packages/temporal/docker/Dockerfile

@@ -0,0 +1,29 @@
+# A worker (or a serve) as its own machine. Running this in containers is what turns "any worker
+# resumes any session" from a claim about processes into a claim about hosts: each of these has its
+# own filesystem, its own hostname, and nothing of the session on disk. What they share is the
+# Temporal cluster and one libSQL store, which is exactly what the README asks an operator to set up.
+
+FROM oven/bun:1.3.14
+
+# python3 and a compiler are here for one dependency: a tree-sitter grammar builds from source at
+# install time. git is for the snapshot packs a worker rebuilds a worktree from: a host that has
+# never seen the project needs it to materialize the tree.
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends git ca-certificates procps curl \
+       python3 make g++ \
+  && rm -rf /var/lib/apt/lists/* \
+  && git config --system user.email opencode@example.com \
+  && git config --system user.name opencode \
+  && git config --system init.defaultBranch main \
+  && git config --system --add safe.directory '*'
+
+WORKDIR /app
+
+COPY package.json bun.lock bunfig.toml tsconfig.json* ./
+COPY patches ./patches
+COPY packages ./packages
+RUN bun install --frozen-lockfile
+
+# The worker by default. The serve role overrides this in compose; both build the same application
+# context, so the only difference is whether an HTTP surface comes with it.
+CMD ["bun", "run", "packages/server/src/worker.ts"]

packages/temporal/docker/compose.yml

@@ -0,0 +1,115 @@
+# Two workers that are two machines, not two processes on one.
+#
+# What they share is what an operator is told to share: one Temporal cluster and one libSQL store.
+# What they do not share is the session's working tree. `worker-a` has the project, `worker-b` gets
+# an empty volume, so a session that moves between them has to rebuild the tree from the snapshot
+# packs in the store. That is the part a single host can never really test, because there the tree
+# is already sitting on the disk the other process is reading.
+#
+#   MODELS_JSON=$(mktemp) && curl -fsS https://models.opencode.ai/api.json -o "$MODELS_JSON"
+#   MODELS_JSON=$MODELS_JSON docker compose -f packages/temporal/docker/compose.yml up -d temporal sqld serve worker-a
+#
+# The catalog comes from a file rather than from the network inside every container: what this
+# stack is for is where a session's files go, and a catalog host having a slow afternoon should not
+# be able to report that as a failure. `scripts/cross-host-check.sh` fetches it for you.
+#
+# Not covered: one libSQL server, so this shows a shared store over a network rather than a store
+# that survives losing a node.
+
+name: opencode-l3
+
+# A mapping rather than a list, because a list cannot be merged: a service that adds one variable
+# would otherwise replace the whole set and silently lose the store.
+x-env: &env
+  OPENCODE_SESSION_EXECUTION: temporal
+  TEMPORAL_ADDRESS: temporal:7233
+  # One store for every host. Without it a session belongs to whichever machine holds its file.
+  OPENCODE_DB_URL: http://sqld:8080
+  OPENCODE_TEMPORAL_STEPPED: "1"
+  OPENCODE_SERVER_PASSWORD: ${OPENCODE_SERVER_PASSWORD:-l3-check}
+  OPENAI_API_KEY: ${OPENAI_API_KEY:?set OPENAI_API_KEY}
+  # The model catalog from a file the script fetched once, rather than from the network inside every
+  # container. What this stack is for is where a session's files go; a catalog host having a slow
+  # afternoon should not be able to report that as a failure.
+  OPENCODE_MODELS_PATH: /models.json
+
+# Each service lists its own volumes in full: a YAML merge key loses to an explicit key, so a list
+# here would be dropped by the service that adds a project volume, models file included.
+x-app: &app
+  image: opencode-temporal:l3
+  depends_on:
+    temporal:
+      condition: service_healthy
+    sqld:
+      condition: service_started
+
+services:
+  temporal:
+    image: temporalio/admin-tools:1.29
+    # The image's own entrypoint is `sleep infinity`, so a command alone becomes arguments to sleep.
+    entrypoint: ["temporal"]
+    command: ["server", "start-dev", "--ip", "0.0.0.0", "--log-level", "warn"]
+    healthcheck:
+      test: ["CMD", "temporal", "operator", "cluster", "health", "--address", "127.0.0.1:7233"]
+      interval: 5s
+      timeout: 5s
+      retries: 40
+
+  sqld:
+    image: ghcr.io/tursodatabase/libsql-server:latest
+    environment:
+      - SQLD_NODE=primary
+    ports:
+      - "8081:8080"
+
+  # Drives workflows, hosts no worker, and is the only thing with an HTTP surface.
+  # The engine's own source is mounted over the copy baked into the image on every service. bun runs
+  # TypeScript directly, so a one-file change does not cost a full dependency install.
+  serve:
+    <<: *app
+    environment:
+      <<: *env
+      OPENCODE_TEMPORAL_ROLE: client
+    # Absolute, because working_dir is the project rather than the checkout: a relative entry path
+    # would be looked for inside the session's tree.
+    command: ["bun", "run", "/app/packages/cli/src/index.ts", "serve", "--port", "4096", "--hostname", "0.0.0.0"]
+    working_dir: /work/project
+    ports:
+      - "4096:4096"
+    volumes:
+      - ../../core/src:/app/packages/core/src:ro
+      - ../src:/app/packages/temporal/src:ro
+      - ${MODELS_JSON:?set MODELS_JSON, or run the check that fetches it}:/models.json:ro
+      - project-a:/work
+
+  worker-a:
+    <<: *app
+    environment:
+      <<: *env
+      OPENCODE_TEMPORAL_ROLE: worker
+    volumes:
+      - ../../core/src:/app/packages/core/src:ro
+      - ../src:/app/packages/temporal/src:ro
+      - ${MODELS_JSON:?set MODELS_JSON, or run the check that fetches it}:/models.json:ro
+      - project-a:/work
+
+  # The volume is mounted one level above the project rather than at it, so the project directory is
+  # an ordinary directory: a host that finds a call never came back can move it out of the way and
+  # build a fresh one, where a mount point can only be refused.
+  #
+  # No project volume of its own that has ever seen this session: an empty tree, so the worktree has
+  # to come from the packs in the store.
+  worker-b:
+    <<: *app
+    environment:
+      <<: *env
+      OPENCODE_TEMPORAL_ROLE: worker
+    volumes:
+      - ../../core/src:/app/packages/core/src:ro
+      - ../src:/app/packages/temporal/src:ro
+      - ${MODELS_JSON:?set MODELS_JSON, or run the check that fetches it}:/models.json:ro
+      - project-b:/work
+
+volumes:
+  project-a:
+  project-b:

packages/temporal/scripts/cross-host-check.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# Any worker resumes any session, across machines rather than across processes.
+#
+# On one host the second worker already has the project on disk, so the interesting half of the
+# claim is never exercised: the tree is there whether or not anything shipped it. Here worker B is a
+# container with an empty project volume, so a session that moves to it has to bring its worktree
+# along, out of the snapshot packs in the shared store.
+#
+# Usage: OPENAI_API_KEY=... packages/temporal/scripts/cross-host-check.sh
+#
+# Not covered: one libSQL server, so this shows a shared store over a network rather than one that
+# survives losing a node.
+
+set -uo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../../.." || exit 1
+COMPOSE="docker compose -f packages/temporal/docker/compose.yml"
+MODEL_ID="${MODEL_ID:-gpt-5-mini}"
+
+fails=0
+ok()  { printf 'PASS %s\n' "$1"; }
+bad() { printf 'FAIL %s   (%s)\n' "$1" "${2:-}"; fails=$((fails + 1)); }
+
+# KEEP=1 leaves the stack up, which is the difference between reading a failure and guessing at it.
+cleanup() { [ -n "${KEEP:-}" ] || $COMPOSE down -v >/dev/null 2>&1; }
+trap cleanup EXIT
+
+[ -n "${OPENAI_API_KEY:-}" ] || { echo "set OPENAI_API_KEY"; exit 1; }
+
+# The model catalog, fetched once here and mounted into every container. Without it each container
+# fetches it itself, with a ten-second timeout, and a catalog host having a slow afternoon reads as
+# the session going wrong. Reuses whatever is already cached on this machine when the fetch does
+# not come back.
+export MODELS_JSON="${MODELS_JSON:-$(mktemp "${TMPDIR:-/tmp}/opencode-models.XXXXXX")}"
+if [ ! -s "$MODELS_JSON" ]; then
+  curl -fsS --max-time 120 "${OPENCODE_MODELS_URL:-https://models.opencode.ai}/api.json" -o "$MODELS_JSON" \
+    || cp "${HOME}/.cache/opencode/models.json" "$MODELS_JSON" 2>/dev/null \
+    || { echo "could not get the model catalog: fetch it to a file and pass MODELS_JSON"; exit 1; }
+fi
+
+# A workspace link pointing at a package the store does not hold. `bun install` leaves these behind
+# when node_modules was pruned by hand, and the first command that needs the package fails in a way
+# that reads like its own bug. Cheap to ask here, and the answer is always to install from a clean
+# tree.
+dangling=""
+for link in packages/*/node_modules/* packages/*/node_modules/@*/* \
+            packages/*/*/node_modules/* packages/*/*/node_modules/@*/*; do
+  [ -L "$link" ] && [ ! -e "$link" ] && dangling="$dangling  $link"$'\n'
+done
+if [ -n "$dangling" ]; then
+  printf 'the install is stale; these workspace links point at nothing:\n%s' "$dangling"
+  echo "run: find . -name node_modules -type d -prune -exec rm -rf {} + && bun install"
+  exit 1
+fi
+
+$COMPOSE down -v >/dev/null 2>&1
+# Built on every run. The compose file mounts `core` and `temporal` over the image, so everything
+# else a worker runs is whatever the image was built from, and a stale image runs current source
+# against an old schema.
+docker build -f packages/temporal/docker/Dockerfile -t opencode-temporal:l3 . >/dev/null \
+  || { echo "build failed"; exit 1; }
+$COMPOSE up -d temporal sqld serve worker-a >/dev/null 2>&1 || { echo "stack failed"; exit 1; }
+
+api() { curl -s -u "opencode:$PW" "$@"; }
+
+# The serve generates its own password on first boot and prefers it over the environment, so ask it
+# rather than tell it.
+for _ in $(seq 1 60); do
+  PW=$($COMPOSE exec -T serve sh -c 'cat /root/.local/state/opencode/password 2>/dev/null' 2>/dev/null | tr -d '\r\n')
+  [ -n "$PW" ] && break
+  sleep 3
+done
+[ -n "$PW" ] && ok "serve is up" || { bad "serve never came up"; exit 1; }
+
+hostA=$($COMPOSE exec -T worker-a hostname 2>/dev/null | tr -d '\r')
+[ -n "$hostA" ] && ok "worker A is a host of its own ($hostA)" || bad "worker A came up"
+
+# A project only worker A and serve can see.
+# The project is a directory under the mounted volume rather than the mount point itself, so a host
+# that finds a call never came back can move it out of the way instead of only refusing it.
+$COMPOSE exec -T serve sh -c \
+  'mkdir -p /work/project && cd /work/project && git init -q && echo hello > README.md && git add -A && git commit -qm init' \
+  >/dev/null 2>&1 && ok "the project is a git checkout on worker A" || { bad "the project fixture failed"; exit 1; }
+
+new_session() {
+  api -X POST http://127.0.0.1:4096/api/session -H 'content-type: application/json' \
+    -d '{"directory":"/work/project"}' | sed -n 's/^{"data":{"id":"\([^"]*\)".*/\1/p'
+}
+prompt() {
+  api -o /dev/null -X POST "http://127.0.0.1:4096/api/session/$1/prompt" \
+    -H 'content-type: application/json' -d "{\"prompt\":{\"text\":$2}}"
+}
+# A turn is over when a step of it ends on "stop", which is not the same as the session leaving the
+# running set: the supervisor stays open for its idle timeout with nothing left to do. Counted
+# rather than matched, because the history of a second turn still contains the first one's ending,
+# and matching would call every later turn finished before it started.
+stops() {
+  local body
+  body=$(api "http://127.0.0.1:4096/api/session/$1/history?limit=100" 2>/dev/null)
+  case "$body" in *InvalidRequestError*) echo "  history rejected: $body" >&2; return 1 ;; esac
+  printf '%s' "$body" | grep -o '"finish":"stop"' | wc -l | tr -d ' '
+}
+await_turn() {
+  local before=$2 now
+  for _ in $(seq 1 90); do
+    now=$(stops "$1") || return 1
+    [ "$now" -gt "$before" ] && return 0
+    sleep 4
+  done
+  return 1
+}
+
+sid=$(new_session)
+[ -n "$sid" ] && ok "a session was created ($sid)" || { bad "no session"; exit 1; }
+api -o /dev/null -X POST "http://127.0.0.1:4096/api/session/$sid/model" \
+  -H 'content-type: application/json' -d "{\"model\":{\"id\":\"$MODEL_ID\",\"providerID\":\"openai\"}}"
+
+# --- turn 1 on worker A: writes a file, so a snapshot of the tree is captured and shipped
+before=$(stops "$sid" || echo 0)
+prompt "$sid" '"Use the bash tool to run exactly: echo TRAVELLED > /work/project/note.txt && cat /work/project/note.txt. Report the output."'
+await_turn "$sid" "$before" && ok "turn 1 finished on worker A" || bad "turn 1 never finished"
+
+packs=$(curl -s http://127.0.0.1:8081/v2/pipeline -H 'content-type: application/json' \
+  -d '{"requests":[{"type":"execute","stmt":{"sql":"select count(*) from snapshot_pack"}},{"type":"close"}]}' \
+  2>/dev/null | grep -o '"value":"[0-9]*"' | head -1 | grep -o '[0-9]*')
+[ "${packs:-0}" -gt 0 ] && ok "the tree was shipped to the shared store ($packs packs)" \
+  || bad "no snapshot packs reached the store" "$packs"
+
+# --- worker A's host goes away, and a host that has never seen this project takes over
+docker kill "$($COMPOSE ps -q worker-a)" >/dev/null 2>&1
+sleep 2
+[ -z "$($COMPOSE ps -q --status running worker-a)" ] && ok "worker A's host is gone" || bad "worker A's host is gone"
+
+$COMPOSE up -d worker-b >/dev/null 2>&1
+hostB=""
+for _ in $(seq 1 30); do
+  hostB=$($COMPOSE exec -T worker-b hostname 2>/dev/null | tr -d '\r')
+  [ -n "$hostB" ] && break
+  sleep 1
+done
+[ "$hostB" != "$hostA" ] && ok "worker B is a different host ($hostB)" || bad "worker B is a different host"
+empty=$($COMPOSE exec -T worker-b sh -c 'ls -A /work/project 2>/dev/null | wc -l' 2>/dev/null | tr -d '\r ')
+[ "${empty:-1}" = "0" ] && ok "worker B's project is empty before the turn" || bad "worker B's project was not empty" "$empty"
+
+# --- turn 2 on worker B: the file only exists there if the worktree travelled
+before=$(stops "$sid" || echo 0)
+prompt "$sid" '"Use the bash tool to run exactly: cat /work/project/note.txt. Report exactly what it printed."'
+await_turn "$sid" "$before" && ok "turn 2 finished on worker B" || bad "turn 2 never finished"
+
+# Asked of worker B's own disk rather than of the transcript. The transcript still holds turn 1,
+# where the file did exist, so anything matched across the whole of it proves nothing about B.
+landed=$($COMPOSE exec -T worker-b sh -c 'cat /work/project/note.txt 2>&1' 2>/dev/null | tr -d '\r')
+case "$landed" in
+  TRAVELLED*) ok "the worktree travelled to worker B" ;;
+  *) bad "the worktree travelled to worker B" "$landed" ;;
+esac
+
+echo
+[ "$fails" -eq 0 ] && echo "cross-host-check: OK" || echo "cross-host-check: $fails failed"
+exit $([ "$fails" -eq 0 ] && echo 0 || echo 1)