From 494742b13e07f401ca6d062423a674a4b8be39db Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 14 Sep 2026 17:35:26 -0700 Subject: [PATCH] Added commands for a session nobody is sitting in front of. `session start` hands over a prompt and returns, `session running` reads the running set off the executor, and `session watch` follows a turn from a machine that never had it. They are plain HTTP clients against any serve in the deployment. The turn now publishes its own durable ending, `session.next.turn.ended`, because a step ending is not a turn ending and a follower in another process has only the replayable stream to read. --- packages/core/src/session/runner/llm.ts | 9 + .../test/session-runner-model-call.test.ts | 21 +- packages/opencode/src/cli/cmd/detached.ts | 264 +++++++++++++++++- packages/opencode/src/cli/cmd/session.ts | 11 +- packages/schema/src/session-event.ts | 19 ++ packages/temporal/README.md | 56 +++- .../scripts/detached-session-check.sh | 171 ++++++++++++ 7 files changed, 545 insertions(+), 6 deletions(-) create mode 100755 packages/temporal/scripts/detached-session-check.sh diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 199c46812fe8..443c1031fdfb 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -514,6 +514,15 @@ const layer = Layer.effect( if (steer) return { continue: true, step: step + 1, promotion: "steer" as SessionInput.Delivery } const queue = yield* SessionInput.hasPending(db, sessionID, "queue") if (queue) return { continue: true, step: 1, promotion: "queue" as SessionInput.Delivery } + // A step ending is not a turn ending, because a steer or a queued prompt continues the same + // turn through another step. This is the one place that knows the turn is over, for both + // modes. Only the ordinary ending: a turn the user stopped, or one a provider error ended, + // does not reach here, so a follower still needs its other reasons to stop waiting. + yield* events.publish(SessionEvent.Turn.Ended, { + sessionID, + timestamp: yield* DateTime.now, + finish: "stop", + }) return { continue: false, step: step + 1, promotion: undefined } }) diff --git a/packages/core/test/session-runner-model-call.test.ts b/packages/core/test/session-runner-model-call.test.ts index 7ee239d68f0d..758decf3ac25 100644 --- a/packages/core/test/session-runner-model-call.test.ts +++ b/packages/core/test/session-runner-model-call.test.ts @@ -34,7 +34,7 @@ import { eq } from "drizzle-orm" import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" import { Auth } from "@opencode-ai/llm/route" import { describe, expect, spyOn } from "bun:test" -import { Cause, DateTime, Deferred, Effect, Exit, Layer, Schema, Stream } from "effect" +import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" import { emptyStep, runnerHarness, seedSession } from "./lib/runner-harness" // One tool call, then a clean finish: the shape a step that wants to keep going produces. @@ -309,6 +309,25 @@ describe("SessionRunner model-only attempt", () => { expect(closed(context)).toBe(true) }), ) + + // A step ending is not a turn ending: a steer or a queued prompt continues the same turn through + // another step, so the one place that decides publishes the turn's own ending. + runnerHarness(textOnly).effect("says the turn ended, once, when nothing follows it", () => + Effect.gen(function* () { + yield* seedSession(sessionID) + const events = yield* EventV2.Service + const ended = yield* events + .subscribe(SessionEvent.Turn.Ended) + .pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped) + const runner = yield* SessionRunner.Service + + yield* runner.runStep({ sessionID, step: 2, promotion: undefined, first: false, force: false }) + + const seen = yield* Fiber.join(ended) + expect(seen.length).toBe(1) + expect(seen[0]?.data.sessionID).toBe(sessionID) + }), + ) }) // Dispatching one recorded call on its own. The policy under test is what happens when a dispatch diff --git a/packages/opencode/src/cli/cmd/detached.ts b/packages/opencode/src/cli/cmd/detached.ts index 8bc7b10633f4..5a091bfd93f4 100644 --- a/packages/opencode/src/cli/cmd/detached.ts +++ b/packages/opencode/src/cli/cmd/detached.ts @@ -1,9 +1,108 @@ -// Commands about the deployment rather than about a session in front of you. `doctor` loads the -// Temporal driver only when it runs, so the rest of the `session` command never pays for it. +// Commands for a session nobody is sitting in front of: start one and walk away, ask the +// deployment what it is still running, and follow one from a machine that never had it. +// +// These are thin HTTP clients on purpose. In a durable deployment the serve processes are +// interchangeable (any of them reads the shared store and signals the same workflows), so a client +// needs an endpoint and a session id, never a particular host. That is the whole reason a session +// can outlive the process that started it, and it is why `start`, `running` and `watch` never talk +// to Temporal. `doctor` does, and loads the driver only when it runs, so the rest of the `session` +// command never pays for it. import type { Argv } from "yargs" import { cmd } from "./cmd" import { UI } from "../ui" +import { ServerAuth } from "@/server/auth" + +const DEFAULT_URL = "http://127.0.0.1:4096" + +type Remote = { readonly url: string; readonly headers: Record } + +function remote(args: { attach?: string; password?: string; username?: string }): Remote { + const url = (args.attach ?? process.env["OPENCODE_SERVER"] ?? DEFAULT_URL).replace(/\/+$/, "") + // No password configured is a valid deployment, so absent auth is absent headers, not an error. + return { url, headers: ServerAuth.headers({ password: args.password, username: args.username }) ?? {} } +} + +async function call(r: Remote, path: string, init?: RequestInit): Promise { + const response = await fetch(`${r.url}/api${path}`, { + ...init, + headers: { ...r.headers, ...(init?.body ? { "content-type": "application/json" } : {}), ...init?.headers }, + }) + if (!response.ok) { + const detail = await response.text().catch(() => "") + throw new Error(`${init?.method ?? "GET"} /api${path} failed: ${response.status} ${detail.slice(0, 200)}`) + } + if (response.status === 204) return undefined as T + const body = (await response.json()) as { data: T } + return body.data +} + +// The remote-facing options every command here shares. Kept in one builder so a second endpoint +// flag can never drift between them. +function remoteOptions(yargs: Argv) { + return yargs + .option("attach", { + type: "string", + describe: `server to talk to (default ${DEFAULT_URL}, or $OPENCODE_SERVER)`, + }) + .option("password", { alias: "p", type: "string", describe: "basic auth password" }) + .option("username", { alias: "u", type: "string", describe: "basic auth username" }) + .option("json", { type: "boolean", describe: "print machine-readable output", default: false }) +} + +interface SessionInfo { + id: string + title?: string + time?: { created?: number; updated?: number } + location?: { directory?: string } +} + +const stamp = (ms?: number) => (ms ? new Date(ms).toISOString().replace("T", " ").slice(0, 19) : "") + +// UI.println writes to stderr, which is right for a person and wrong for a pipe. Anything a script +// is meant to read goes to stdout instead. +const emit = (line: string) => process.stdout.write(line + "\n") + +export const SessionStartCommand = cmd({ + command: "start ", + describe: "start a session, hand it a prompt, and return without waiting for it", + builder: (yargs: Argv) => + remoteOptions(yargs) + .positional("prompt", { type: "string", describe: "what the agent should do", demandOption: true }) + .option("dir", { type: "string", describe: "session working directory (default: this one)" }) + .option("model", { type: "string", describe: "provider/model, e.g. openai/gpt-5-mini" }), + handler: async (args) => { + const r = remote(args) + try { + const directory = args.dir ?? process.cwd() + const session = await call(r, "/session", { + method: "POST", + body: JSON.stringify({ directory }), + }) + if (args.model) { + const slash = args.model.indexOf("/") + if (slash < 1) throw new Error(`--model wants provider/model, got ${args.model}`) + const model = { providerID: args.model.slice(0, slash), id: args.model.slice(slash + 1) } + await call(r, `/session/${session.id}/model`, { method: "POST", body: JSON.stringify({ model }) }) + } + // The prompt is admitted, not awaited. Whoever is polling the task queue runs the turn, and + // this process has nothing left to do with it. + await call(r, `/session/${session.id}/prompt`, { + method: "POST", + body: JSON.stringify({ prompt: { text: args.prompt } }), + }) + if (args.json) { + emit(JSON.stringify({ id: session.id, directory, url: r.url })) + return + } + emit(session.id) + UI.println(` follow it with: opencode session watch ${session.id}`) + } catch (error) { + UI.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + } + }, +}) // What this process resolved, and what is wrong with it. Deploying was a handful of variables that // have to agree, with no way to ask whether they did: every mistake in them fails as something @@ -29,3 +128,164 @@ export const SessionDoctorCommand = cmd({ UI.println("this deployment looks consistent") }, }) + +export const SessionRunningCommand = cmd({ + command: "running", + describe: "list the sessions this deployment is executing right now", + builder: (yargs: Argv) => remoteOptions(yargs), + handler: async (args) => { + const r = remote(args) + try { + // Which sessions are running is the executor's answer, not a guess from the transcript: a + // durable deployment reads it from the running workflows, so it survives a restart of + // whichever process happens to be answering this call. + const active = await call>(r, "/session/active") + const ids = Object.keys(active) + if (args.json) { + emit(JSON.stringify(ids.map((id) => ({ id, status: active[id]?.type })))) + return + } + if (ids.length === 0) { + UI.println("nothing running") + return + } + const all = await call(r, "/session").catch(() => [] as SessionInfo[]) + const byId = new Map(all.map((s) => [s.id, s])) + for (const id of ids) { + const session = byId.get(id) + const cells = [id, active[id]?.type ?? "?", stamp(session?.time?.updated), session?.title ?? ""] + emit(cells.join(" ")) + } + } catch (error) { + UI.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + } + }, +}) + +// What a follower prints. The stream carries far more than a person watching wants to read, so this +// keeps the events that say the work moved and drops the token-level ones. +const INTERESTING: Record string | undefined> = { + "session.next.prompted": () => "prompted", + "session.next.step.started": () => "step", + "session.next.tool.called": (d) => `tool ${d.tool}: ${JSON.stringify(d.input ?? {}).slice(0, 120)}`, + "session.next.tool.success": (d) => `tool ok ${firstText(d.content).slice(0, 200)}`, + "session.next.tool.failed": (d) => `tool failed ${firstText(d.content).slice(0, 200)}`, + "session.next.text.ended": (d) => (d.text ? `said: ${String(d.text).slice(0, 400)}` : undefined), + "session.next.step.failed": (d) => `step failed: ${d.error?.message ?? ""}`, +} + +function firstText(content: unknown): string { + if (!Array.isArray(content)) return "" + const part = content.find((c) => c && typeof c === "object" && (c as any).type === "text") as any + return part?.text ? String(part.text).trim() : "" +} + +export const SessionWatchCommand = cmd({ + command: "watch ", + describe: "follow a running session from anywhere, and exit when its turn ends", + builder: (yargs: Argv) => + remoteOptions(yargs) + .positional("sessionID", { type: "string", describe: "session to follow", demandOption: true }) + .option("wait", { type: "boolean", default: true, describe: "keep following until the turn ends" }), + handler: async (args) => { + const r = remote(args) + const sessionID = args.sessionID + + // The turn saying so itself. A step ending is not a turn ending, because a steer or a queued + // prompt continues the same turn, so this is the one event that can end a follow on its own. + const turnEnded = (event: { type?: string }) => event.type === "session.next.turn.ended" + + // The event above cannot answer every case. A turn the user stopped, or one a provider error + // ended, publishes no ordinary ending, and the running set holds a session for its whole idle + // timeout after the work is done, so it is slow. It is also the one reading that only ever says + // "finished", which is why it can end a watch without being able to hang one. + const POLL_MS = Number(process.env.OPENCODE_WATCH_POLL_MS ?? 30_000) + const inactive = async () => { + const active = await call>(r, "/session/active").catch(() => undefined) + // Unreachable is not finished, which is the whole point of this command. + return active !== undefined && !(sessionID in active) + } + + // The stream ends when the serve this is attached to restarts, which is the event this command + // exists for. Exiting 0 there reports a turn that is still running as done. + const attempts = 30 + try { + for (let attempt = 0; ; attempt++) { + const response = await fetch(`${r.url}/api/session/${sessionID}/event`, { + headers: r.headers, + }).catch(() => undefined) + if (!response?.ok || !response.body) { + if (attempt >= attempts) throw new Error(`cannot follow ${sessionID}: ${response?.status ?? "unreachable"}`) + await new Promise((resolve) => setTimeout(resolve, 1000)) + continue + } + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = "" + let ended = false + // Held across iterations rather than started fresh each time: a read that loses the race is + // still queued on the stream, and dropping it drops whatever it goes on to deliver. + let pending: ReturnType | undefined + for (;;) { + const next = (pending ??= reader.read()) + let timer: ReturnType | undefined + const chunk = args.wait + ? await Promise.race([ + next, + new Promise<"quiet">((resolve) => { + timer = setTimeout(() => resolve("quiet"), POLL_MS) + }), + ]) + : await next + if (timer) clearTimeout(timer) + if (chunk === "quiet") { + if (await inactive()) { + ended = true + break + } + continue + } + pending = undefined + const { done, value } = chunk + if (done) break + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const raw of lines) { + const line = raw.startsWith("data:") ? raw.slice(5).trim() : raw.trim() + if (!line.startsWith("{")) continue + let event: { type?: string; data?: any } + try { + event = JSON.parse(line) + } catch { + continue + } + if (args.json) { + emit(line) + } else { + const render = event.type ? INTERESTING[event.type] : undefined + const text = render?.(event.data ?? {}) + if (text) UI.println(`${stamp(event.data?.timestamp)} ${text}`) + } + if (args.wait && turnEnded(event)) { + ended = true + break + } + } + if (ended) break + } + await reader.cancel().catch(() => {}) + // Without `--wait` the stream itself is the whole command, so its end is this one's too. + if (ended || !args.wait) return + // The stream dropped with the turn still going. Reconnect and keep following. + if (attempt >= attempts) throw new Error(`lost the stream for ${sessionID}`) + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + } catch (error) { + UI.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + } + }, +}) diff --git a/packages/opencode/src/cli/cmd/session.ts b/packages/opencode/src/cli/cmd/session.ts index 13befb6e9eed..d0df40554ab7 100644 --- a/packages/opencode/src/cli/cmd/session.ts +++ b/packages/opencode/src/cli/cmd/session.ts @@ -1,7 +1,7 @@ import type { Argv } from "yargs" import { Effect } from "effect" import { cmd } from "./cmd" -import { SessionDoctorCommand } from "./detached" +import { SessionDoctorCommand, SessionRunningCommand, SessionStartCommand, SessionWatchCommand } from "./detached" import { effectCmd, fail } from "../effect-cmd" import { Session } from "@/session/session" import { SessionID } from "../../session/schema" @@ -46,7 +46,14 @@ export const SessionCommand = cmd({ command: "session", describe: "manage sessions", builder: (yargs: Argv) => - yargs.command(SessionListCommand).command(SessionDeleteCommand).command(SessionDoctorCommand).demandCommand(), + yargs + .command(SessionListCommand) + .command(SessionDeleteCommand) + .command(SessionStartCommand) + .command(SessionRunningCommand) + .command(SessionWatchCommand) + .command(SessionDoctorCommand) + .demandCommand(), async handler() {}, }) diff --git a/packages/schema/src/session-event.ts b/packages/schema/src/session-event.ts index 3a559c3e38a4..cab8f1ebe2c9 100644 --- a/packages/schema/src/session-event.ts +++ b/packages/schema/src/session-event.ts @@ -145,6 +145,24 @@ export namespace Shell { export type Ended = typeof Ended.Type } +// The turn, as opposed to the steps it was made of. A step ending is not a turn ending: a steer or +// a queued prompt continues the same turn through another step, so a follower cannot tell the two +// apart from a finish reason. Durable, because the follower that needs it is another process +// reading the replayable stream, which is built from the rows and nothing else. +export namespace Turn { + export const Ended = Event.define({ + type: "session.next.turn.ended", + ...options, + schema: { + ...Base, + // What the last step of the turn came to, so a follower can say why it stopped rather than + // only that it did. + finish: Schema.String, + }, + }) + export type Ended = typeof Ended.Type +} + export namespace Step { export const Started = Event.define({ type: "session.next.step.started", @@ -458,6 +476,7 @@ export const DurableDefinitions = Event.inventory( Step.Started, Step.Ended, Step.Failed, + Turn.Ended, Text.Started, Text.Ended, Tool.Input.Started, diff --git a/packages/temporal/README.md b/packages/temporal/README.md index 4c699e0199db..e3f7565ce908 100644 --- a/packages/temporal/README.md +++ b/packages/temporal/README.md @@ -66,6 +66,7 @@ session runs as the workflow `session-exec-`. | `OPENCODE_TEMPORAL_API_KEY`, `OPENCODE_TEMPORAL_API_KEY_FILE` | | Temporal Cloud credentials, the second read from a file. | | `OPENCODE_TEMPORAL_TLS_CERT`, `OPENCODE_TEMPORAL_TLS_KEY`, `OPENCODE_TEMPORAL_TLS_CA` | | A certificate pair for a cluster with mTLS. `OPENCODE_TEMPORAL_TLS=1` for TLS without a client certificate. | | `OPENCODE_EVENT_POLL_MS` | | How often a live subscriber re-reads the log for events another process appended. `0` turns the tick off. | +| `OPENCODE_WATCH_POLL_MS` | 30000 | How often `session watch` asks the running set whether the turn ended in a gap it could not see. | | `OPENCODE_DB` | | One absolute path shared by every process on a host. | | `OPENCODE_DB_URL`, `OPENCODE_DB_AUTH_TOKEN` | | A libSQL URL for a store shared across hosts. Takes precedence over `OPENCODE_DB`. | @@ -264,7 +265,35 @@ destroys work: a tree is moved only when a host-local note (`snapshot/tip.ts`) s behind the store, so a host holding a capture that never shipped is left alone. Packs are ordered by their chain, not by `time_created`, because hosts do not agree on the time. -## Picking a deployment rather than assembling one +## A session that outlives its client + +The pieces above make a session survive a worker. Together they make it survive the client: start +something, close the laptop, and pick it up from a machine that has never seen it. A session is a +workflow rather than a process, the running set comes from Temporal visibility, the store is +shared, and a live tail re-reads. What was missing was a way to say so from a command line: + +```bash +# hand over a prompt and walk away; prints the session id and exits +opencode session start "port the auth module to the new API" --attach http://gateway:4096 + +# what this deployment is running right now, across every client that ever connected +opencode session running --attach http://gateway:4096 + +# follow one from anywhere, and stop when the turn stops +opencode session watch ses_abc123 --attach http://gateway:4096 +``` + +`--attach` takes any serve in the deployment, because they are interchangeable: each reads the same +store and signals the same workflows. `$OPENCODE_SERVER` sets the endpoint once. For an interactive +terminal rather than a follower, `opencode attach --session ` puts the TUI on the session. + +`start`, `running` and `watch` are plain HTTP clients. `watch` exits when the runner publishes the +turn's own ending, `session.next.turn.ended`, which is durable so that a process on another host +can read it from the replayable stream. A turn the user stopped or a provider error ended publishes +no ordinary ending, so `watch` also asks the running set on a slow poll, the one reading that can +end a follow without being able to hang one. + +### Picking a deployment rather than assembling one The settings are not independent, and getting them wrong fails as something else later: a store only one process can see reads as a worker that never picks anything up. `OPENCODE_TEMPORAL_PROFILE` @@ -289,3 +318,28 @@ TEMPORAL_ADDRESS=temporal.internal:7233 \ OPENCODE_TEMPORAL_TLS_CERT=/run/secrets/tls.crt \ OPENCODE_TEMPORAL_TLS_KEY=/run/secrets/tls.key # a cluster with mTLS ``` + +### What recovers on its own, and what needs a person + +| Failure | Local executor | Temporal executor | +|---|---|---| +| Process dies mid-turn | Nothing restarts it; a later `resume` reads the record | The step's activities retry. A started pinned tool call stays with its host; its step is closed on the shared queue and the turn goes on | +| Tool outcome is absent | Completed results are kept, declared-idempotent tools run again, other started calls are reported unknown | The same runner rule | +| Two dispatches read one pending call | One coordinator serializes its own turn | One deterministic admission event admits one dispatch; idempotent tools may run again | +| Attempts overlap | One process | Same-run owner tokens fence later appends; a step's tools and seal share their model call's token | +| User stops the turn | The coordinator cancels its run | The drain is cancelled and the supervisor keeps serving; a declined permission is a halt, not a failed tool | +| Restart during an approval or question | The pending row is kept; execution has to be restarted | A retried ask adopts the row and can be answered from another process | +| A behind host publishes files | Not reachable | Refused: the pack store checks the chain and the owner token | +| A fresh worker receives a project | Needs the directory | Packs rebuild the tracked files at the recorded path; ignored files do not travel | +| A turn never stops stepping | The coordinator's own loop | The supervisor stops after 200 steps and says so | + +An unknown tool outcome is a loss of evidence, not proof that the tool stopped or failed. The model +can ask for a new call after reading it. A non-idempotent external effect needs an idempotency key, +an outcome query or a person to decide what happened. + +### Checks that need real processes + +`scripts/detached-session-check.sh` runs the client story against a Temporal dev server, one +standalone worker, two serve processes and one shared store: a turn started on serve A survives A +being killed mid-tool, serve B replays it, and `start`, `running` and `watch` work from a cold +client. It needs an OpenAI key and does not run in CI. diff --git a/packages/temporal/scripts/detached-session-check.sh b/packages/temporal/scripts/detached-session-check.sh new file mode 100755 index 000000000000..42a7512e15c0 --- /dev/null +++ b/packages/temporal/scripts/detached-session-check.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Proves the claim a durable session is supposed to make: it belongs to the deployment, not to +# whoever started it. One worker, two serve processes, one shared store, and a client that is only +# ever a client. +# +# 1. serve A starts a turn, then A is killed while a tool is still running +# 2. the turn finishes anyway, on a worker that is a separate process +# 3. serve B, which never saw the session, reports it running and replays the whole transcript +# 4. `session start` hands over a prompt and returns, holding no terminal +# 5. `session watch` follows that turn live from a cold client and stops when the turn stops +# +# Needs: bun, the temporal CLI, GNU coreutils (`timeout`), and an OpenAI key. Nothing here is a +# unit test; it is the evidence for a claim that only shows up across processes. +# +# Usage: OPENAI_API_KEY=... packages/temporal/scripts/detached-session-check.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +OC="$ROOT/packages/opencode/src/index.ts" +RUN="${RUN_DIR:-${TMPDIR:-/tmp}/opencode-l3}" +PORT_TEMPORAL="${PORT_TEMPORAL:-7240}" +PORT_A="${PORT_A:-4610}" +PORT_B="${PORT_B:-4611}" +MODEL="${MODEL:-openai/gpt-5-mini}" + +fails=0 +ok() { printf 'PASS %s\n' "$1"; } +bad() { printf 'FAIL %s (%s)\n' "$1" "${2:-}"; fails=$((fails + 1)); } + +pids=() +cleanup() { + for pid in "${pids[@]:-}"; do + [ -n "$pid" ] || continue + kill -9 $(pgrep -P "$pid" 2>/dev/null) "$pid" 2>/dev/null + done +} +trap cleanup EXIT + +[ -n "${OPENAI_API_KEY:-}" ] || { echo "set OPENAI_API_KEY"; exit 1; } + +# A workspace link pointing at a package the store does not hold. `bun install` leaves these behind +# when node_modules was pruned by hand, and the first command that needs the package fails in a way +# that reads like its own bug. Cheap to ask here, and the answer is always to install from a clean +# tree. +dangling="" +for link in "$ROOT"/packages/*/node_modules/* "$ROOT"/packages/*/node_modules/@*/* \ + "$ROOT"/packages/*/*/node_modules/* "$ROOT"/packages/*/*/node_modules/@*/*; do + [ -L "$link" ] && [ ! -e "$link" ] && dangling="$dangling $link"$'\n' +done +if [ -n "$dangling" ]; then + printf 'the install is stale; these workspace links point at nothing:\n%s' "$dangling" + echo "run: find . -name node_modules -type d -prune -exec rm -rf {} + && bun install" + exit 1 +fi + +rm -rf "$RUN"; mkdir -p "$RUN/proj" "$RUN/logs" +git -C "$RUN/proj" init -q +echo hello > "$RUN/proj/README.md" +git -C "$RUN/proj" add -A +git -C "$RUN/proj" -c user.email=a@b.c -c user.name=t commit -qm init + +export OPENCODE_SESSION_EXECUTION=temporal +export TEMPORAL_ADDRESS="127.0.0.1:$PORT_TEMPORAL" +# One store both serves and the worker read. This is what makes any process able to answer for any +# session; without it a session belongs to the host holding its file. +export OPENCODE_DB="$RUN/shared.db" +export OPENCODE_TEMPORAL_STEPPED=1 +# A stored password wins over the environment for the v2 serve, so a script that invents one gets +# 401 on every call. Take what the server will actually be asking for. +STORED="${XDG_STATE_HOME:-$HOME/.local/state}/opencode/password" +if [ -f "$STORED" ]; then + OPENCODE_SERVER_PASSWORD="$(cat "$STORED")" +else + OPENCODE_SERVER_PASSWORD="${OPENCODE_SERVER_PASSWORD:-l3-check}" +fi +export OPENCODE_SERVER_PASSWORD + +temporal server start-dev --port "$PORT_TEMPORAL" --ui-port $((PORT_TEMPORAL + 1000)) --log-level warn \ + > "$RUN/logs/temporal.log" 2>&1 & +pids+=($!) +for _ in $(seq 1 30); do + temporal operator cluster health --address "127.0.0.1:$PORT_TEMPORAL" >/dev/null 2>&1 && break + sleep 1 +done + +OPENCODE_TEMPORAL_ROLE=worker bun run "$ROOT/packages/server/src/worker.ts" > "$RUN/logs/worker.log" 2>&1 & +pids+=($!) + +cd "$RUN/proj" || exit 1 +OPENCODE_TEMPORAL_ROLE=client bun run "$ROOT/packages/cli/src/index.ts" serve --port "$PORT_A" \ + > "$RUN/logs/serveA.log" 2>&1 & +serveA=$!; pids+=($serveA) +OPENCODE_TEMPORAL_ROLE=client bun run "$ROOT/packages/cli/src/index.ts" serve --port "$PORT_B" \ + > "$RUN/logs/serveB.log" 2>&1 & +pids+=($!) + +A="http://127.0.0.1:$PORT_A" +B="http://127.0.0.1:$PORT_B" +AUTH="opencode:$OPENCODE_SERVER_PASSWORD" + +# Bounded, because a fixed sleep is either a slow script or a flaky one. Both serves boot a whole +# application context, which on a cold module cache is not quick. +# Answering at all is not enough: an unauthorized answer is still an answer, and treating it as +# ready turns a credentials problem into a confusing timeout later. +wait_for() { + for _ in $(seq 1 60); do + [ "$(curl -s -o /dev/null -w '%{http_code}' -u "$AUTH" "$1/api/session")" = "200" ] && return 0 + sleep 2 + done + return 1 +} +wait_for "$A" && wait_for "$B" || { echo "serves never came up; see $RUN/logs"; exit 1; } + +# The id of the session, not of anything nested in it: the field is read off the first line of the +# document, so a later `"id"` (a model, a message) cannot be picked up instead. +session_id() { sed -n 's/^{"data":{"id":"\([^"]*\)".*/\1/p' | head -1; } + +# --- 1. a turn started on serve A, long enough to still be running when A dies +created=$(curl -s -u "$AUTH" -X POST "$A/api/session" -H 'content-type: application/json' \ + -d "{\"directory\":\"$RUN/proj\"}") +sid=$(printf '%s' "$created" | session_id) +[ -n "$sid" ] && ok "serve A created a session" || { bad "serve A created a session" "$created"; exit 1; } + +provider=${MODEL%%/*}; model=${MODEL#*/} +curl -s -o /dev/null -u "$AUTH" -X POST "$A/api/session/$sid/model" -H 'content-type: application/json' \ + -d "{\"model\":{\"id\":\"$model\",\"providerID\":\"$provider\"}}" +curl -s -o /dev/null -u "$AUTH" -X POST "$A/api/session/$sid/prompt" -H 'content-type: application/json' \ + -d '{"prompt":{"text":"Use the bash tool to run exactly: sleep 40 && echo SURVIVED. Then report the output."}}' +sleep 18 +pgrep -f "sleep 40 && echo SURVIVED" > /dev/null && ok "the tool is running on the worker" \ + || bad "the tool is running on the worker" "it never started" + +# --- 2. kill the process that started it, mid-tool +kill -9 $(pgrep -P $serveA 2>/dev/null) $serveA 2>/dev/null +sleep 3 +[ -z "$(lsof -nP -iTCP:$PORT_A -sTCP:LISTEN 2>/dev/null)" ] && ok "serve A is gone" || bad "serve A is gone" +pgrep -f "sleep 40 && echo SURVIVED" > /dev/null && ok "the turn outlived the client that started it" \ + || bad "the turn outlived the client that started it" "the tool died with serve A" + +# --- 3. serve B, which never saw this session, knows it and can replay it +running=$(curl -s -u "$AUTH" "$B/api/session/active") +case "$running" in *"$sid"*) ok "serve B reports it running" ;; *) bad "serve B reports it running" "$running" ;; esac + +sleep 35 +timeout 30 curl -s -N -u "$AUTH" "$B/api/session/$sid/event" > "$RUN/logs/replay.txt" 2>&1 +grep -q "SURVIVED" "$RUN/logs/replay.txt" && ok "serve B replays work done while no client existed" \ + || bad "serve B replays work done while no client existed" + +# --- 4. start a turn and walk away +started=$(timeout 90 bun run "$OC" session start \ + "Use the bash tool to run exactly: sleep 20 && echo WATCHED. Then report the output." \ + --attach "$B" --model "$MODEL" --dir "$RUN/proj" --json 2>/dev/null) +sid2=$(printf '%s' "$started" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p') +[ -n "$sid2" ] && ok "session start returned an id without waiting" || bad "session start returned an id" "$started" + +listed=$(timeout 60 bun run "$OC" session running --attach "$B" --json 2>/dev/null) +case "$listed" in *"$sid2"*) ok "session running lists it" ;; *) bad "session running lists it" "$listed" ;; esac + +# --- 5. follow it live from a client that has never seen it, and stop when the turn stops +began=$(date +%s) +timeout 120 bun run "$OC" session watch "$sid2" --attach "$B" > "$RUN/logs/watch.txt" 2>&1 +took=$(( $(date +%s) - began )) +grep -q "WATCHED" "$RUN/logs/watch.txt" && ok "session watch followed the turn" \ + || bad "session watch followed the turn" "$(tail -3 "$RUN/logs/watch.txt")" +[ "$took" -lt 100 ] && ok "session watch stopped when the turn did (${took}s)" \ + || bad "session watch stopped when the turn did" "${took}s, so it hung" + +echo +[ "$fails" -eq 0 ] && echo "detached-session-check: OK" || echo "detached-session-check: $fails failed" +exit $([ "$fails" -eq 0 ] && echo 0 || echo 1)