diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 9afc34cdef7c..5db6e53a6323 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -350,4 +350,7 @@ an outcome query or a person to decide what happened. `scripts/detached-session-check.sh` runs the client story against a Temporal dev server, one standalone worker, two serve processes and one shared store: a turn started on serve A survives A being killed mid-tool, serve B replays it, and `start`, `running`, `watch` and `schedule` work from -a cold client. It needs an OpenAI key and does not run in CI. +a cold client. `scripts/cross-host-check.sh` runs the worktree story in containers +(`docker/compose.yml`): a session writes a file on worker A, A's host is killed, and worker B, whose +project volume is empty, continues the session and reads the file back. Both need an OpenAI key, +and neither runs in CI. diff --git a/packages/temporal/docker/Dockerfile b/packages/temporal/docker/Dockerfile new file mode 100644 index 000000000000..757a07a128c0 --- /dev/null +++ b/packages/temporal/docker/Dockerfile @@ -0,0 +1,29 @@ +# A worker (or a serve) as its own machine. Running this in containers is what turns "any worker +# resumes any session" from a claim about processes into a claim about hosts: each of these has its +# own filesystem, its own hostname, and nothing of the session on disk. What they share is the +# Temporal cluster and one libSQL store, which is exactly what the README asks an operator to set up. + +FROM oven/bun:1.3.14 + +# python3 and a compiler are here for one dependency: a tree-sitter grammar builds from source at +# install time. git is for the snapshot packs a worker rebuilds a worktree from: a host that has +# never seen the project needs it to materialize the tree. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates procps curl \ + python3 make g++ \ + && rm -rf /var/lib/apt/lists/* \ + && git config --system user.email opencode@example.com \ + && git config --system user.name opencode \ + && git config --system init.defaultBranch main \ + && git config --system --add safe.directory '*' + +WORKDIR /app + +COPY package.json bun.lock bunfig.toml tsconfig.json* ./ +COPY patches ./patches +COPY packages ./packages +RUN bun install --frozen-lockfile + +# The worker by default. The serve role overrides this in compose; both build the same application +# context, so the only difference is whether an HTTP surface comes with it. +CMD ["bun", "run", "packages/server/src/worker.ts"] diff --git a/packages/temporal/docker/compose.yml b/packages/temporal/docker/compose.yml new file mode 100644 index 000000000000..79f9f9c3c756 --- /dev/null +++ b/packages/temporal/docker/compose.yml @@ -0,0 +1,115 @@ +# Two workers that are two machines, not two processes on one. +# +# What they share is what an operator is told to share: one Temporal cluster and one libSQL store. +# What they do not share is the session's working tree. `worker-a` has the project, `worker-b` gets +# an empty volume, so a session that moves between them has to rebuild the tree from the snapshot +# packs in the store. That is the part a single host can never really test, because there the tree +# is already sitting on the disk the other process is reading. +# +# MODELS_JSON=$(mktemp) && curl -fsS https://models.opencode.ai/api.json -o "$MODELS_JSON" +# MODELS_JSON=$MODELS_JSON docker compose -f packages/temporal/docker/compose.yml up -d temporal sqld serve worker-a +# +# The catalog comes from a file rather than from the network inside every container: what this +# stack is for is where a session's files go, and a catalog host having a slow afternoon should not +# be able to report that as a failure. `scripts/cross-host-check.sh` fetches it for you. +# +# Not covered: one libSQL server, so this shows a shared store over a network rather than a store +# that survives losing a node. + +name: opencode-l3 + +# A mapping rather than a list, because a list cannot be merged: a service that adds one variable +# would otherwise replace the whole set and silently lose the store. +x-env: &env + OPENCODE_SESSION_EXECUTION: temporal + TEMPORAL_ADDRESS: temporal:7233 + # One store for every host. Without it a session belongs to whichever machine holds its file. + OPENCODE_DB_URL: http://sqld:8080 + OPENCODE_TEMPORAL_STEPPED: "1" + OPENCODE_SERVER_PASSWORD: ${OPENCODE_SERVER_PASSWORD:-l3-check} + OPENAI_API_KEY: ${OPENAI_API_KEY:?set OPENAI_API_KEY} + # The model catalog from a file the script fetched once, rather than from the network inside every + # container. What this stack is for is where a session's files go; a catalog host having a slow + # afternoon should not be able to report that as a failure. + OPENCODE_MODELS_PATH: /models.json + +# Each service lists its own volumes in full: a YAML merge key loses to an explicit key, so a list +# here would be dropped by the service that adds a project volume, models file included. +x-app: &app + image: opencode-temporal:l3 + depends_on: + temporal: + condition: service_healthy + sqld: + condition: service_started + +services: + temporal: + image: temporalio/admin-tools:1.29 + # The image's own entrypoint is `sleep infinity`, so a command alone becomes arguments to sleep. + entrypoint: ["temporal"] + command: ["server", "start-dev", "--ip", "0.0.0.0", "--log-level", "warn"] + healthcheck: + test: ["CMD", "temporal", "operator", "cluster", "health", "--address", "127.0.0.1:7233"] + interval: 5s + timeout: 5s + retries: 40 + + sqld: + image: ghcr.io/tursodatabase/libsql-server:latest + environment: + - SQLD_NODE=primary + ports: + - "8081:8080" + + # Drives workflows, hosts no worker, and is the only thing with an HTTP surface. + # The engine's own source is mounted over the copy baked into the image on every service. bun runs + # TypeScript directly, so a one-file change does not cost a full dependency install. + serve: + <<: *app + environment: + <<: *env + OPENCODE_TEMPORAL_ROLE: client + # Absolute, because working_dir is the project rather than the checkout: a relative entry path + # would be looked for inside the session's tree. + command: ["bun", "run", "/app/packages/cli/src/index.ts", "serve", "--port", "4096", "--hostname", "0.0.0.0"] + working_dir: /work/project + ports: + - "4096:4096" + volumes: + - ../../core/src:/app/packages/core/src:ro + - ../src:/app/packages/temporal/src:ro + - ${MODELS_JSON:?set MODELS_JSON, or run the check that fetches it}:/models.json:ro + - project-a:/work + + worker-a: + <<: *app + environment: + <<: *env + OPENCODE_TEMPORAL_ROLE: worker + volumes: + - ../../core/src:/app/packages/core/src:ro + - ../src:/app/packages/temporal/src:ro + - ${MODELS_JSON:?set MODELS_JSON, or run the check that fetches it}:/models.json:ro + - project-a:/work + + # The volume is mounted one level above the project rather than at it, so the project directory is + # an ordinary directory: a host that finds a call never came back can move it out of the way and + # build a fresh one, where a mount point can only be refused. + # + # No project volume of its own that has ever seen this session: an empty tree, so the worktree has + # to come from the packs in the store. + worker-b: + <<: *app + environment: + <<: *env + OPENCODE_TEMPORAL_ROLE: worker + volumes: + - ../../core/src:/app/packages/core/src:ro + - ../src:/app/packages/temporal/src:ro + - ${MODELS_JSON:?set MODELS_JSON, or run the check that fetches it}:/models.json:ro + - project-b:/work + +volumes: + project-a: + project-b: diff --git a/packages/temporal/scripts/cross-host-check.sh b/packages/temporal/scripts/cross-host-check.sh new file mode 100755 index 000000000000..dfa336564bdd --- /dev/null +++ b/packages/temporal/scripts/cross-host-check.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# Any worker resumes any session, across machines rather than across processes. +# +# On one host the second worker already has the project on disk, so the interesting half of the +# claim is never exercised: the tree is there whether or not anything shipped it. Here worker B is a +# container with an empty project volume, so a session that moves to it has to bring its worktree +# along, out of the snapshot packs in the shared store. +# +# Usage: OPENAI_API_KEY=... packages/temporal/scripts/cross-host-check.sh +# +# Not covered: one libSQL server, so this shows a shared store over a network rather than one that +# survives losing a node. + +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." || exit 1 +COMPOSE="docker compose -f packages/temporal/docker/compose.yml" +MODEL_ID="${MODEL_ID:-gpt-5-mini}" + +fails=0 +ok() { printf 'PASS %s\n' "$1"; } +bad() { printf 'FAIL %s (%s)\n' "$1" "${2:-}"; fails=$((fails + 1)); } + +# KEEP=1 leaves the stack up, which is the difference between reading a failure and guessing at it. +cleanup() { [ -n "${KEEP:-}" ] || $COMPOSE down -v >/dev/null 2>&1; } +trap cleanup EXIT + +[ -n "${OPENAI_API_KEY:-}" ] || { echo "set OPENAI_API_KEY"; exit 1; } + +# The model catalog, fetched once here and mounted into every container. Without it each container +# fetches it itself, with a ten-second timeout, and a catalog host having a slow afternoon reads as +# the session going wrong. Reuses whatever is already cached on this machine when the fetch does +# not come back. +export MODELS_JSON="${MODELS_JSON:-$(mktemp "${TMPDIR:-/tmp}/opencode-models.XXXXXX")}" +if [ ! -s "$MODELS_JSON" ]; then + curl -fsS --max-time 120 "${OPENCODE_MODELS_URL:-https://models.opencode.ai}/api.json" -o "$MODELS_JSON" \ + || cp "${HOME}/.cache/opencode/models.json" "$MODELS_JSON" 2>/dev/null \ + || { echo "could not get the model catalog: fetch it to a file and pass MODELS_JSON"; exit 1; } +fi + +# A workspace link pointing at a package the store does not hold. `bun install` leaves these behind +# when node_modules was pruned by hand, and the first command that needs the package fails in a way +# that reads like its own bug. Cheap to ask here, and the answer is always to install from a clean +# tree. +dangling="" +for link in packages/*/node_modules/* packages/*/node_modules/@*/* \ + packages/*/*/node_modules/* packages/*/*/node_modules/@*/*; do + [ -L "$link" ] && [ ! -e "$link" ] && dangling="$dangling $link"$'\n' +done +if [ -n "$dangling" ]; then + printf 'the install is stale; these workspace links point at nothing:\n%s' "$dangling" + echo "run: find . -name node_modules -type d -prune -exec rm -rf {} + && bun install" + exit 1 +fi + +$COMPOSE down -v >/dev/null 2>&1 +# Built on every run. The compose file mounts `core` and `temporal` over the image, so everything +# else a worker runs is whatever the image was built from, and a stale image runs current source +# against an old schema. +docker build -f packages/temporal/docker/Dockerfile -t opencode-temporal:l3 . >/dev/null \ + || { echo "build failed"; exit 1; } +$COMPOSE up -d temporal sqld serve worker-a >/dev/null 2>&1 || { echo "stack failed"; exit 1; } + +api() { curl -s -u "opencode:$PW" "$@"; } + +# The serve generates its own password on first boot and prefers it over the environment, so ask it +# rather than tell it. +for _ in $(seq 1 60); do + PW=$($COMPOSE exec -T serve sh -c 'cat /root/.local/state/opencode/password 2>/dev/null' 2>/dev/null | tr -d '\r\n') + [ -n "$PW" ] && break + sleep 3 +done +[ -n "$PW" ] && ok "serve is up" || { bad "serve never came up"; exit 1; } + +hostA=$($COMPOSE exec -T worker-a hostname 2>/dev/null | tr -d '\r') +[ -n "$hostA" ] && ok "worker A is a host of its own ($hostA)" || bad "worker A came up" + +# A project only worker A and serve can see. +# The project is a directory under the mounted volume rather than the mount point itself, so a host +# that finds a call never came back can move it out of the way instead of only refusing it. +$COMPOSE exec -T serve sh -c \ + 'mkdir -p /work/project && cd /work/project && git init -q && echo hello > README.md && git add -A && git commit -qm init' \ + >/dev/null 2>&1 && ok "the project is a git checkout on worker A" || { bad "the project fixture failed"; exit 1; } + +new_session() { + api -X POST http://127.0.0.1:4096/api/session -H 'content-type: application/json' \ + -d '{"directory":"/work/project"}' | sed -n 's/^{"data":{"id":"\([^"]*\)".*/\1/p' +} +prompt() { + api -o /dev/null -X POST "http://127.0.0.1:4096/api/session/$1/prompt" \ + -H 'content-type: application/json' -d "{\"prompt\":{\"text\":$2}}" +} +# A turn is over when a step of it ends on "stop", which is not the same as the session leaving the +# running set: the supervisor stays open for its idle timeout with nothing left to do. Counted +# rather than matched, because the history of a second turn still contains the first one's ending, +# and matching would call every later turn finished before it started. +stops() { + local body + body=$(api "http://127.0.0.1:4096/api/session/$1/history?limit=100" 2>/dev/null) + case "$body" in *InvalidRequestError*) echo " history rejected: $body" >&2; return 1 ;; esac + printf '%s' "$body" | grep -o '"finish":"stop"' | wc -l | tr -d ' ' +} +await_turn() { + local before=$2 now + for _ in $(seq 1 90); do + now=$(stops "$1") || return 1 + [ "$now" -gt "$before" ] && return 0 + sleep 4 + done + return 1 +} + +sid=$(new_session) +[ -n "$sid" ] && ok "a session was created ($sid)" || { bad "no session"; exit 1; } +api -o /dev/null -X POST "http://127.0.0.1:4096/api/session/$sid/model" \ + -H 'content-type: application/json' -d "{\"model\":{\"id\":\"$MODEL_ID\",\"providerID\":\"openai\"}}" + +# --- turn 1 on worker A: writes a file, so a snapshot of the tree is captured and shipped +before=$(stops "$sid" || echo 0) +prompt "$sid" '"Use the bash tool to run exactly: echo TRAVELLED > /work/project/note.txt && cat /work/project/note.txt. Report the output."' +await_turn "$sid" "$before" && ok "turn 1 finished on worker A" || bad "turn 1 never finished" + +packs=$(curl -s http://127.0.0.1:8081/v2/pipeline -H 'content-type: application/json' \ + -d '{"requests":[{"type":"execute","stmt":{"sql":"select count(*) from snapshot_pack"}},{"type":"close"}]}' \ + 2>/dev/null | grep -o '"value":"[0-9]*"' | head -1 | grep -o '[0-9]*') +[ "${packs:-0}" -gt 0 ] && ok "the tree was shipped to the shared store ($packs packs)" \ + || bad "no snapshot packs reached the store" "$packs" + +# --- worker A's host goes away, and a host that has never seen this project takes over +docker kill "$($COMPOSE ps -q worker-a)" >/dev/null 2>&1 +sleep 2 +[ -z "$($COMPOSE ps -q --status running worker-a)" ] && ok "worker A's host is gone" || bad "worker A's host is gone" + +$COMPOSE up -d worker-b >/dev/null 2>&1 +hostB="" +for _ in $(seq 1 30); do + hostB=$($COMPOSE exec -T worker-b hostname 2>/dev/null | tr -d '\r') + [ -n "$hostB" ] && break + sleep 1 +done +[ "$hostB" != "$hostA" ] && ok "worker B is a different host ($hostB)" || bad "worker B is a different host" +empty=$($COMPOSE exec -T worker-b sh -c 'ls -A /work/project 2>/dev/null | wc -l' 2>/dev/null | tr -d '\r ') +[ "${empty:-1}" = "0" ] && ok "worker B's project is empty before the turn" || bad "worker B's project was not empty" "$empty" + +# --- turn 2 on worker B: the file only exists there if the worktree travelled +before=$(stops "$sid" || echo 0) +prompt "$sid" '"Use the bash tool to run exactly: cat /work/project/note.txt. Report exactly what it printed."' +await_turn "$sid" "$before" && ok "turn 2 finished on worker B" || bad "turn 2 never finished" + +# Asked of worker B's own disk rather than of the transcript. The transcript still holds turn 1, +# where the file did exist, so anything matched across the whole of it proves nothing about B. +landed=$($COMPOSE exec -T worker-b sh -c 'cat /work/project/note.txt 2>&1' 2>/dev/null | tr -d '\r') +case "$landed" in + TRAVELLED*) ok "the worktree travelled to worker B" ;; + *) bad "the worktree travelled to worker B" "$landed" ;; +esac + +echo +[ "$fails" -eq 0 ] && echo "cross-host-check: OK" || echo "cross-host-check: $fails failed" +exit $([ "$fails" -eq 0 ] && echo 0 || echo 1)