diff --git a/demo/e2e/run.ts b/demo/e2e/run.ts index 86fcae6e..0db351ae 100644 --- a/demo/e2e/run.ts +++ b/demo/e2e/run.ts @@ -183,14 +183,18 @@ const SCENARIOS: Scenario[] = [ // the role read out of it, the acceptor and the dial — rather than in // anything either of them covers. soloResumeSync, - // AND ITS ONE-SIDED SIBLING, expected RED: solo-resume-sync proves - // the both-sides reload recovers, and solo.ts (~2988-3012) documents - // in prose that the ONE-SIDED reload — the acceptor reloads, the - // reader keeps a stale handle that `conn-status` will never - // invalidate — does not. This scenario turns that prose into a gate: - // it asserts the recovery as if it worked, fails today, and the - // xfail machinery FAILS THE SUITE the day an engine fix makes it - // pass un-promoted (drop its `expected` flag then). + // AND ITS ONE-SIDED SIBLING, GREEN SINCE #113: solo-resume-sync proves + // the both-sides reload recovers; this one proves the harder half — + // only the ACCEPTOR reloads, and the reader, holding a handle to a page + // that no longer exists, has to notice and dial again. It was pinned + // `expected: "red"` from PR #108 until #113 landed a `conn-status` that + // reports a wire that DIED (the `gone:` marker) and a solo-page + // wire-keeper that re-dials on it — and only on it, so a healthy peer + // is never double-dialled. Measured heal: 35s, three runs, dominated by + // the pinned endpoint's QUIC idle timeout (nothing under the reader's + // side dies when a peer merely vanishes, so the wire ends on a timer + // rather than on an error). It runs here, after the resume family, + // because its preconditions are exactly theirs. oneSidedReload, soloEphemeral, // THE WORKER HOST'S STORAGE EGRESS (STORAGE-EGRESS.md's T-E): the same @@ -288,14 +292,28 @@ const SCENARIOS: Scenario[] = [ // failure much easier to read. soloRecovery, soloRecoveryFile, - // THE TWO RELAY-PARTITION PINS, both expected RED and both SLOW — - // each spends minutes proving a heal that does not come (the - // freshly-paired ceremony wires never re-dial, and `conn-status` - // never learns a wire died; the full trace is in the scenarios' own - // banners). They run this late so a suite that is already broken - // earlier never pays for them, and after the solo family because - // their preconditions (pairing, convergence, the harness's own fault - // levers) are all claims made green above. + // THE TWO RELAY-PARTITION SCENARIOS, green since #113 landed end to + // end and the last of the `expected: "red"` pins from PR #108 to be + // dropped. Between them they claim that a paired account survives its + // RELAY going away — vanishing for everyone, and vanishing for one + // device while the other stays connected — with both pages open the + // whole time and nobody reloading, re-pairing or pressing anything. + // Three waves of gap had to close for that: the ceremony wires that + // never re-dialled and the `conn-status` latch that made re-dialling + // unsafe (#113 as filed), then the page's wire-keeper, then the stale + // transport chain in the engine that the wire-keeper uncovered. The + // scenarios' own banners carry all three, and demo/host's + // `conn-gone-check.ts` and `rebind-sync-check.ts` are the headless + // gates under them. + // + // STILL LATE, AND STILL FOR THE OLD REASON, though the cost has + // collapsed: ~39s each now (it was ~184s while each spent its whole + // HEAL_MS proving a heal that never came), and what remains is two + // deliberate 30s windows per scenario — the control crossing's bound + // and the negative assertion that the partition is real. That is still + // the most expensive pair in the suite, so they run after everything + // whose failure would explain theirs: pairing, convergence, and the + // harness's own fault levers are all claims made green above. relayPartition, relayPartitionAsym, // The erase ceremony: seeds a name, a petname and a storage sentinel, diff --git a/demo/e2e/scenarios/convergence-soak.ts b/demo/e2e/scenarios/convergence-soak.ts index 8a2f6202..868bda79 100644 --- a/demo/e2e/scenarios/convergence-soak.ts +++ b/demo/e2e/scenarios/convergence-soak.ts @@ -31,12 +31,15 @@ // bring-up and then a 45 s cadence, and §3 gives it a 20 s trailing // debounced flush armed by its own mutations — so two devices bound to // one account's bucket converge THROUGH THE STORE whatever state their -// peer wires are in. That matters because of the gap -// scenarios/relay-partition.ts pins (`expected: "red"`): relay wires -// between two LIVE paired pages never re-dial after a relay outage — -// a ceremony-time wire is not re-entered, and `conn-status` never -// learns the wire died. This soak therefore NEVER asserts relay-path -// recovery. It stops and starts the relay because that is a fault a +// peer wires are in. That independence was originally forced by a gap +// (relay wires never re-dialled after an outage — the three-wave +// history now lives in scenarios/relay-partition.ts's banner, green +// since #113 landed end to end), but it remains the right design even +// with the wires healing themselves in ~5s: the soak's oracle must not +// depend on BOTH channels being healthy, because faulting one of them +// is half its alphabet. This soak therefore still NEVER asserts +// relay-path recovery — relay-partition{,-asym} own that claim. It +// stops and starts the relay because that is a fault a // real user's network produces, and it demands only what the bucket // can carry. // diff --git a/demo/e2e/scenarios/one-sided-reload.ts b/demo/e2e/scenarios/one-sided-reload.ts index c6fc6b86..12f79291 100644 --- a/demo/e2e/scenarios/one-sided-reload.ts +++ b/demo/e2e/scenarios/one-sided-reload.ts @@ -1,105 +1,124 @@ -// A DOCUMENTED, UNRECOVERED GAP, TURNED INTO A GATE — the sibling -// `solo-resume-sync` deliberately does not claim, and this scenario is -// what watches for the day it becomes true. +// THE HARDER HALF OF A RELOAD: only ONE side comes back, and the side +// that stayed up is the one that has to notice. // -// THE CLAIM UNDER TEST, stated by solo.ts itself (~2988-3012, next to the -// resume-wire code): when only ONE side of a paired account reloads, and -// that side is the one that ACCEPTS (the adder), the recovery does not -// happen. The acceptor's own resume is fine — it posts a fresh acceptor -// and waits, correctly. Its peer — the reader, who DIALS — is the -// problem: it still holds a connection HANDLE from before the reload, -// and has no way to learn the thing on the other end of it is gone. -// `conn-status` reports the handshake's outcome once and is never -// invalidated afterwards (engine/guest/src/lib.rs:4407 writes the -// outcome into `conn_results` once per connection, and :4413-4420's -// `conn_status` reads that same entry back forever), so the reader sees -// a "healthy" connection to a page that no longer exists and never -// re-dials. The honest fix belongs in the engine — a `conn-status` that -// goes false when the connection drops — and until it lands, this file -// cannot tell the difference between a healthy peer and a departed one. -// Filed as #113. +// ─── THE CLAIM ─────────────────────────────────────────────────────── // -// WHY THIS IS A GATE RATHER THAN MORE PROSE: `solo-resume-sync` proves -// the BOTH-sides-reload case recovers (the ordinary one — a user closes -// their laptop, then their phone) and explicitly declines to claim this -// one, with a paragraph pointing here. A paragraph does not fail a CI -// run when the gap closes. This scenario asserts the RECOVERY as if it -// already held, marked `expected: "red"` (see run.ts ~95-105): today it -// times out and is recorded `ok (xfail: expected red)` — a known gap, -// tracked, not hidden. The day the engine grows a live `conn-status` -// and B's resume loop re-dials, this scenario turns green, which the -// harness reads as the xfail FLIPPING — the whole suite fails until -// someone drops the `expected: "red"` flag. That failure IS the -// promotion notice. +// Two devices of one account are paired and syncing. The ADDER — the +// side that ACCEPTS — reloads for real: page gone, SharedWorker gone, +// engine gone, a fresh acceptor posted on the other side of the reload. +// The READER never reloads and is never touched. A todo authored on the +// reader afterwards still reaches the adder, because the reader NOTICES +// that the connection it has been holding is dead and dials again. // -// WHICH DIRECTION, traced rather than assumed: A is the adder (it -// accepts) and B is the reader (it dials), per the ceremony's fixed -// direction (solo.ts ~2564-2570, issue #78). Only A reloads. After A's +// ─── IT WAS RED, AND WHY (kept, because it is the point) ───────────── +// +// Pinned `expected: "red"` from PR #108 until #113 closed it, and the +// gap had two halves that had to close together: +// +// * THE READER HAD NO SIGNAL. `conn-status` reported the outcome of +// the HANDSHAKE and was never invalidated afterwards: the engine +// wrote it into `conn_results` once and read that entry back for +// ever. So the reader's handle to a page that had ceased to exist +// answered exactly as a live one did. Nothing in the reader's code +// could ask "is this still good?", because there was no answer. +// * AND NO TRIGGER. The reader was in a CEREMONY role (`joinerWire`), +// which runs at most once; the only patient retry on the page, +// `resumeWire`, was entered exclusively from the resumed-boot +// branch — and the reader is precisely the side that did not +// reload. Adding a re-dial timer without the first half would have +// been the double-dial #78's direction discipline exists to +// prevent: a second connection carrying the same subscriptions, +// silently. +// +// WHAT CLOSED IT, both halves of #113: +// +// * THE ENGINE now gives every connection a monitor that awaits the +// iroh WIT's `wait-closed` and overwrites that connection's entry +// with `Err("gone: …")`. The `gone:` prefix is a machine-readable +// marker, spelled out in engine.wit's `conn-status` doc comment; it +// overwrites only an `Ok`, so a handshake failure — which is more +// informative — survives. +// * THE PAGE now runs ONE WIRE-KEEPER for the life of every paired +// page rather than only on resumed boots (demo/host/solo.ts's +// `wireKeeper`). Both ceremonies hand it their wire when they are +// done; each tick it reads `conn-status` on the handles it holds, +// and a `gone:` — and nothing else — clears that peer and lets the +// next tick re-dial. The double-dial discipline is unchanged; it is +// merely enforceable now. +// +// ─── THE MEASURED NUMBER, AND WHAT GOVERNS IT ──────────────────────── +// +// MEASURED: 35.0s from the reader's todo to the adder holding it, three +// consecutive runs, varying by tens of milliseconds. Its shape: +// +// ~30s the wire actually ending. THIS IS THE WHOLE COST, and it +// belongs to the pinned endpoint rather than to either side's +// code: when a PEER VANISHES, nothing under the surviving +// side dies. Its socket is fine, the relay is fine, and no +// close frame is ever sent by a page that has been navigated +// away from — so the connection ends on the QUIC IDLE +// TIMEOUT, not on an error. (Contrast the relay-death shape, +// where the relay leg is a websocket over TCP and the +// endpoint learns synchronously: measured under a second in +// demo/host/conn-gone-check.ts. Same marker, two mechanisms, +// two orders of magnitude apart.) +// + ≤5s the keeper's next tick noticing (`KEEPER_TICK_MS`) +// + <1s the re-dial and the re-subscription +// ------ +// ~35s +// +// `REWIRE` below is that with roughly 2x of margin. Sizing it much +// tighter would gate the pinned endpoint's idle timeout, which is not +// this repo's number to hold; sizing it much looser would make a real +// regression cost minutes per suite run before it reported. +// +// ─── WHICH DIRECTION, traced rather than assumed ───────────────────── +// +// A is the adder (it accepts) and B is the reader (it dials), per the +// ceremony's fixed direction (issue #78). Only A reloads. After A's // resume: // - A's role is "writer accepts": it reposts a fresh acceptor and -// genuinely waits — for ITS OWN us-devices directory to show a -// child dialling in, and for that dial to land. That side of the -// resume is honestly correct; it is just listening into silence, -// because nothing ever dials it again. -// - B's role is "reader dials", but B never reloads, so B's -// resume-wire — the only code path that would make B re-dial — -// never runs at all. B's ORIGINAL ceremony-time connection object -// is still sitting there, and `conn-status` on it still answers - // with the handshake's old, one-time-written "true" - // (engine/guest/src/lib.rs:4407/:4413-4420). B has no symptom to -// act on and no trigger to re-dial: nothing in B's code ever asks -// "is this still good?" once the handshake result is latched. -// So the crossing asserted is B → A: a todo authored on B after A's -// reload has no path to A, because the only way it could travel — a -// fresh dial from B, prompted by B noticing its peer came back — never -// happens. A's freshly-reposted acceptor is real and waiting; nobody -// ever dials it. That is the cleanest single claim, and the one this -// scenario asserts: the reader side is where the gap actually lives -// (it is the side with no signal and no retry), so the reader's silence -// is what the xfail act below drives and waits on. -// +// waits — for its own us-devices directory to show a child dialling +// in, and for that dial to land. That side was always correct; it +// was just listening into silence. +// - B's role is "reader dials", and B never reloads. B's keeper — +// armed at the end of B's own joining ceremony, which is the change +// — sweeps its one outbound handle every tick, sees the `gone:` +// marker when the idle timeout fires, forgets the wire and the +// subscription that rode on it, and dials A's fresh acceptor. +// So the crossing asserted is B → A: it is the direction that requires +// the reader to have noticed, which is the whole claim. // RELAY-ONLY ACCOUNT, ON PURPOSE — CONTRACT: no storage is bound // anywhere in this scenario, and nothing here must add one. With a // bucket bound, the worker's pull cadence (SYNC.md §2, ~45s) would -// eventually deliver A's todo to B through the bucket regardless of the -// relay wire, and a scenario that could pass via a channel it never -// meant to exercise is not testing what its banner says. With no -// storage, the RELAY subduction is the only channel either device has, -// so a todo that fails to cross is unambiguously the stale-handle gap -// and not a masked, slower success. -// -// THE BOUNDED WAIT'S ARITHMETIC: the resume loop retries on a 5s cadence -// (solo.ts's `RESUME_TICK_MS`). A healthy re-wire (in the fixed world) -// would need at most a few ticks to notice a directory entry and dial — -// well under a minute. `REWIRE` below is sized generously past that -// (comfortably containing several retry cycles plus relay/CI slack) -// without turning a permanently-red scenario into a multi-minute tax on -// every suite run: it never actually gets there today, because nothing -// on B's side loops at all. +// eventually deliver the todo through the bucket regardless of the relay +// wire, and a scenario that can pass via a channel it never meant to +// exercise is not testing what its banner says. With no storage, the +// RELAY subduction is the only channel either device has, so the +// crossing below is unambiguously the re-dial. import type { Page } from "npm:playwright@1.57.0"; import type { Ctx, Scenario } from "../run.ts"; import { act, assert, assertEquals, SOLO_KEYS, waitForBoot } from "../util.ts"; import { addTodo, createAccount, pairPages, solo, todoRows, until, WAITS } from "../solo-util.ts"; -/** How long to wait for the crossing that the gap says will not happen. - * Sized to comfortably contain several of the resume loop's 5s retry - * ticks plus relay/CI slack — see the arithmetic note above. Kept short - * on purpose: this scenario runs red in every suite invocation until the - * engine grows a live `conn-status`, and a five-minute tax on every run - * for a known, documented gap would be a worse citizen than a tight - * bound that still gives the healthy world (once it exists) room to - * land inside it. */ +/** How long the crossing is given. Measured 35.0s (three runs); see the + * banner's arithmetic — ~30s of it is the pinned endpoint's QUIC idle + * timeout, which is what the reader is waiting on, plus one ≤5s keeper + * tick and a sub-second re-dial. 75s is that with roughly 2x of margin: + * enough that a busy CI box cannot turn the endpoint's own timer into a + * red, tight enough that a genuine regression reports in about a minute + * instead of costing the suite five. */ const REWIRE = 75_000; const scenario: Scenario = { name: "one-sided-reload", why: - "REGRESSION GATE for a documented gap (solo.ts ~2988-3012): after only the ADDER side of a " + - "paired account reloads, the READER never learns its connection handle is stale and a todo " + - "never crosses — expected red until the engine invalidates conn-status on disconnect", - expected: "red", + "when only the ADDER side of a paired account reloads, the READER notices its connection went " + + "`gone:` and re-dials by itself — a todo authored on the reader still crosses, with no " + + "ceremony and nothing reloaded on that side (was red until #113: engine gone-marker + the " + + "page's wire-keeper)", // Comfortably contains REWIRE plus the pairing/convergence beats ahead // of it; the suite-wide default is sized for scenarios with no // multi-tick relay wait at all. @@ -170,49 +189,69 @@ const scenario: Scenario = { assertEquals(st.resumed, true, "A's engine RESUMED rather than starting fresh"); }); - // --- THE XFAIL CLAIM: asserted as if the engine fix already existed -- + // --- THE CLAIM: the reader notices, and dials again ----------------- await act( - "xfail: a todo authored on B after A's reload reaches A within a resume-sized window", + "a todo authored on B after A's reload reaches A: B noticed its wire went gone and re-dialled", async () => { - // THE DIRECTION TRACED ABOVE: B → A is the crossing the gap - // provably blocks. A's resume has reposted a fresh acceptor and - // is genuinely waiting (solo.ts's writer-accepts resume path); - // what never happens is B re-dialling it, because B's - // resume-wire never runs — B never reloaded, so B's original - // connection object is still there, and `conn-status` on it - // still answers with the handshake's old, one-time "true" - // (engine/guest/src/lib.rs:4407 writes it, :4413-4420 reads it - // back). Filed as #113. + // THE DIRECTION TRACED ABOVE: B → A is the crossing that requires + // the reader to have noticed. A's resume has reposted a fresh + // acceptor and is waiting (solo.ts's writer-accepts path); what + // used to never happen is B re-dialling it. Now B's wire-keeper + // sweeps its handle every tick, and when the pinned endpoint's + // idle timeout finally ends the connection to a page that no + // longer exists, `conn-status` says `gone:` and the next tick + // dials. #113. await addTodo(pageB, "call the bank"); - // DIAGNOSIS RIDES WITH THE FAILURE: on a red run this wait times - // out, and what a future reader needs is not "it timed out" but - // the STALE-HANDLE SHAPE — both sides' boot traces, both sides' - // device-status/conn-adjacent fields, and both sides' todo lists - // — so the failure teaches the gap rather than hiding it behind - // a bare deadline. + // DIAGNOSIS RIDES WITH THE FAILURE, and it is worth MORE now than + // it was as an xfail: this act is green, so the next reader to + // meet it as a failure is meeting a REGRESSION, and what they + // need is which half went. `wireHealth` is the keeper's own + // account of the handles it holds — B reading "alive" on a wire + // to a page that is gone means the engine's monitor stopped + // firing; B reading "gone" with no re-dial behind it means the + // keeper stopped keeping. Both sides' boot traces and todo lists + // are kept for the same reason they always were. let diag: unknown = null; const titles = await until([pageA, pageB], "B's new todo on A", async () => { diag = { aTrace: await solo(pageA, "bootTrace").catch((e) => String(e)), aStatus: await solo(pageA, "deviceStatus").catch((e) => String(e)), + aWire: await solo(pageA, "wireHealth").catch((e) => String(e)), aTodos: await solo(pageA, "todos").catch((e) => String(e)), bTrace: await solo(pageB, "bootTrace").catch((e) => String(e)), bStatus: await solo(pageB, "deviceStatus").catch((e) => String(e)), + bWire: await solo(pageB, "wireHealth").catch((e) => String(e)), bTodos: await solo(pageB, "todos").catch((e) => String(e)), }; const t = (await solo(pageA, "todos").catch(() => [])) as string[]; return t.includes("call the bank") ? t : false; }, REWIRE).catch((e) => { throw new Error( - `${(e as Error).message}; the stale-handle shape at timeout: ${JSON.stringify(diag)}`, + `${(e as Error).message}; the reader never re-dialled within ${REWIRE / 1000}s ` + + `(measured heal is 35s — see this file's banner). Both sides at timeout: ` + + `${JSON.stringify(diag)}`, ); }); assert( titles.some((t) => t.includes("buy milk")), `A kept the pre-reload row too: ${JSON.stringify(titles)}`, ); + // AND ON THE ROWS, not only in the engine: the user's complaint + // is about a screen. A heal that reached the partition but not + // the surface is still a device that "stopped updating". + // + // WAITED FOR RATHER THAN SAMPLED, and the difference is real: the + // assertion above reads the ENGINE, and the app's rows are + // re-rendered off a later drain — so reading the DOM in the same + // breath catches A mid-frame and fails on a row that appears a + // beat later. `WAITS.converge` here is a paint deadline, not a + // network one; the network wait already happened. + await until([pageA, pageB], "B's new todo RENDERED on A", async () => { + const rows = await todoRows(pageA).allTextContents(); + return rows.some((t) => t.includes("call the bank")); + }, WAITS.converge); }, ); }, diff --git a/demo/e2e/scenarios/relay-partition-asym.ts b/demo/e2e/scenarios/relay-partition-asym.ts index 4503e069..8120889b 100644 --- a/demo/e2e/scenarios/relay-partition-asym.ts +++ b/demo/e2e/scenarios/relay-partition-asym.ts @@ -1,19 +1,21 @@ // ONE DEVICE'S RELAY PATH IS CUT — the asymmetric partition, and the -// heal that does not come. Registered `expected: "red"`: like its -// symmetric sibling (scenarios/relay-partition.ts), this scenario PINS -// A GAP rather than asserting a working property. +// heal that comes by itself. Green since #113 landed end to end, +// alongside its symmetric sibling (scenarios/relay-partition.ts), whose +// banner carries the full three-wave history both files were red for. // // ─── WHY A SECOND FILE ────────────────────────────────────────────── // -// `expected: "red"` is a whole-SCENARIO flag: the run stops at the -// first failing act, so two partitions in one file would mean the -// second was never reached and never pinned. These are two distinct -// failures of the same missing machinery — a relay that vanishes for -// EVERYONE, and a relay path that vanishes for ONE device while the -// other sits happily connected — and each deserves to go green -// independently on the day it can. +// A scenario stops at its first failing act, so two partitions in one +// file would mean the second was never reached — and while both were +// pinned `expected: "red"` that also meant the second was never pinned. +// These are two distinct shapes of the same fault — a relay that +// vanishes for EVERYONE, and a relay path that vanishes for ONE device +// while the other sits happily connected — and they went green +// independently, which is exactly the argument for having kept them +// apart. (Measured: this one flipped first, on runs where its sibling +// was still losing a repaint race.) // -// ─── THE CLAIM IT WOULD MAKE IF THE GAP WERE CLOSED ───────────────── +// ─── THE CLAIM ────────────────────────────────────────────────────── // // A is on the harness's real relay; B reaches the same relay through a // severable TCP proxy, which is B's `?relay=` and therefore B's home @@ -74,38 +76,58 @@ // afresh. That is precisely the state the missing machinery would have // to act in. // -// ─── WHERE THE GAP IS ─────────────────────────────────────────────── +// ─── WHAT HAD TO BE TRUE, AND THE THREE WAVES IT TOOK ─────────────── // -// The same one relay-partition.ts traces in full, reached by a -// different road; the short form, in the page's own terms: +// relay-partition.ts's banner traces all three in full; this file +// reaches the same machinery by a different road, so the short form, +// with the one thing that is specific to THIS fault called out: // -// * the only patient retry on this page is `resumeWire`, entered ONLY -// from the resumed-boot branch (demo/host/solo.ts:3366, :3374), and -// it is the sole caller of `rebindEndpoint` for a `Closed` endpoint -// (solo.ts:3132); -// * a page that paired in THIS session is in a ceremony role -// (`joinerWire` solo.ts:2673 / `adderWire` :2900), each latched by -// its own `…Wired` guard after the first success — a retry for -// wiring that never came up, not a re-dial for a wire that died; -// * and neither side can learn the wire died at all: `conn-status` -// latches the handshake's outcome and is never invalidated -// — the outcome goes into `conn_results` once and is never -// removed (engine/guest/src/lib.rs:4407, with :4343/:4400 for the -// error outcomes), and `conn-status` reads it back for ever -// (:4413-4420). solo.ts:2988-3012 already writes this down for the -// neighbouring one-sided-reload case, though by the latch's OLD -// line numbers — the code moved, the fact did not. +// WAVE 1 — THE PAGE COULD NOT RETRY, AND COULD NOT HAVE (#113 as +// filed). The ceremony wires (`joinerWire` / `adderWire`) latch on +// first success and never re-dial; the only patient loop was entered +// from the resumed-boot branch alone. And `conn-status` latched the +// HANDSHAKE's outcome for ever, so no page could tell a dead wire from +// a live one — which made any re-dial-on-a-timer the double-dial the +// direction discipline forbids (#78). // -// THE MISSING PIECE, in the engine's own terms: a `conn-status` that -// goes false when the connection drops (named as the honest fix at -// solo.ts:3009-3012 — filed as #113), and a page-side retry armed for -// the life of the page rather than only on the resumed-boot path. - +// WAVE 2 — THE PAGE HALF: a machine-readable `gone:` marker written by +// a per-connection monitor on the iroh WIT's `wait-closed`, plus ONE +// WIRE-KEEPER in solo.ts armed for the life of every paired page, +// which re-dials on that marker and on nothing else — rebinding a +// `Closed` endpoint on the way, because a dead relay path latches this +// device's own transport shut as well as its connections. +// +// AND THIS FAULT IS WHERE THAT SIGNAL IS AT ITS STRONGEST: `sever()` +// kills B's relay socket outright, so B's death is an ERROR rather +// than silence. That was the argument for the fault shape while this +// file was red (a heal that fails even here fails for want of +// machinery, not information), and it is why B's side of the heal is +// the fast one now. +// +// WAVE 3 — THE STALE TRANSPORT CHAIN, which wave 2 uncovered: a +// `QueueTransport` that held its own channel ends and so could never +// fail, a subduction teardown driven only by that failure, an +// `add_connection` that APPENDS rather than replaces, and a +// `sync_with_peer` that walks the resulting list in order under a +// never-firing timeout — so after a rebind it parked on the dead +// connection and never reached the live one. The gone-monitor now +// closes the dead connection's inbound queues, so subduction's own +// teardown runs. Pinned red→green by demo/host/rebind-sync-check.ts +// (`just rebind-sync`): post-rebind `sync-start` settles in 0.20s, +// crossing 0.03–0.06s both ways. +// +// THE ASYMMETRY THAT WAVE 3 EXPLAINS is worth keeping in THIS file in +// particular, because this scenario is the asymmetric one and the two +// asymmetries are unrelated: only the side that CALLS `sync_with_peer` +// walked the stale list, so the dialling side hung while the accepting +// side settled normally. That has nothing to do with which device's +// relay path was cut — B is the cut device here AND the dialling one, +// which is why this fault showed the bug so cleanly. // // NOTHING RELOADS HERE, and no storage is bound — see relay-partition.ts -// for both, in full. A reload would enter `resumeWire` and heal (that is -// `solo-resume-sync`'s green claim); a bound store would let a bucket -// carry the todos and make the relay claim unfalsifiable. +// for both, in full. A reload would re-handshake from nothing and hide +// what is being claimed; a bound store would let a bucket carry the +// todos and make the relay claim unfalsifiable. import type { Page } from "npm:playwright@1.57.0"; import type { Ctx, Scenario } from "../run.ts"; @@ -144,13 +166,31 @@ const CONTROL_MS = 30_000; * roughly four thousand times over. */ const CUT_WATCH_MS = 30_000; -/** What the heal is given. Derived exactly as relay-partition.ts's is - * (see that file for the arithmetic): RESUME_TICK_MS 5s + the 30s dial - * `until` + the 30s subscribe `until` + a 45s relay pull cadence ≈ 110s, - * rounded to 150s for two full attempts. The probe behind this file - * watched the full 150s and saw nothing move, so the red is not this - * number being tight. */ -const HEAL_MS = 150_000; +/** What the heal is given, sized as relay-partition.ts's is and for the + * same reasons (that file carries the arithmetic in full): + * + * ≤5s the wire-keeper's tick noticing and running its repair + * + ~1s rebind, repost, re-dial, re-handshake — measured headless at + * 0.20s to settle and 0.03–0.06s to cross + * (demo/host/rebind-sync-check.ts) + * + slack for a missed tick and a busy box + * ------ + * MEASURED end to end here: 4.3–4.6s over five consecutive runs, + * from `proxy.restore()` to both ENGINES holding the merged set. + * + * 60s is that with an order of magnitude of margin. The old 150s was + * sized for a heal that was never coming, and that reasoning is stale. */ +const HEAL_MS = 60_000; + +/** And what the SCREENS are given once the engines agree. A repaint is + * not a claim violation: the rows are rendered off a drain that runs a + * beat behind the engine, so a bare DOM read here raced the paint and + * was this file's last intermittent red. 30s is thirty of that drain's + * 1s cadence — generous against a loaded box, short enough that a screen + * which genuinely never updates still reports promptly. Same figure and + * same reasoning as relay-partition.ts's; measured here at 0.03–0.53s + * over five runs. */ +const RENDER_MS = 30_000; /** A page's whole story, for attaching to a wait that lost — the * solo-offline-sync pattern. Best-effort: a diagnosis that threw would @@ -167,6 +207,14 @@ async function diagnose(label: string, page: Page): Promise { await read("todos", () => solo(page, "todos")), await read("usSynced", () => solo(page, "usSynced")), await read("sync", () => solo(page, "syncStatus")), + // THE WIRE-KEEPER'S OWN VIEW (#113): which handles the page holds + // and what `conn-status` says about each. First thing to read on a + // failure, because it says WHICH wave came back: two LIVE wires with + // nothing crossing is wave 3 (subduction on a stale connection); a + // dead, missing or never-re-dialled wire is wave 2 (the page's + // wire-keeper); a wire still reading alive on a path that has been + // severed for half a minute is wave 1 (the `gone:` marker). + await read("wire", () => solo(page, "wireHealth")), // A PREFIX, not the whole id: 64 hex characters per page would // drown the two facts either side of it, and all a reader needs // from a transport address here is whether it is present and @@ -189,9 +237,14 @@ async function engineTodos(page: Page): Promise { const scenario: Scenario = { name: "relay-partition-asym", why: - "cutting ONE device's relay path and healing it brings both devices back into sync by itself (XFAIL: it does not; see this file's banner)", - expected: "red", - deadlineMs: 420_000, + "cutting ONE device's relay path and healing it brings both devices back into sync by itself — the cut device marks its wire gone, rebinds, re-dials and resumes subduction, with no reload and no ceremony", + // Boot + pairing across the two relay URLs + the crossing + the 30s + // CUT_WATCH_MS window + a ~4.4s heal ≈ 38s measured (37.8–38.7s over + // five runs). 180s is over + // 4x that; the two deliberate 30s waits are what this scenario costs, + // and a regressed heal reports inside HEAL_MS long before this outer + // deadline is reached. + deadlineMs: 180_000, // PAGE A stays on the harness's own relay — `baseQuery` gives it that // for free, so this scenario names no relay at all here. B's override // is the whole asymmetry and it is written where B is opened. @@ -341,7 +394,7 @@ const scenario: Scenario = { await input.fill(""); }); - // --- THE CLAIM (this is the act that is red) ------------------------ + // --- THE CLAIM, in two acts ----------------------------------------- await act("B's path is restored and BOTH devices converge, with nothing reloaded", async () => { // `restore()` heals NEW connections only — the ones that lived @@ -366,12 +419,28 @@ const scenario: Scenario = { `on both.\n ${await diagnoseBoth(pageA, pageB)}`, ); }); + }); + + await act("and both SCREENS follow: each device RENDERS the other's cut-off edit", async () => { + // The user's own test — a partition that healed in the engine and + // not on the surface is still a device that "stopped updating". + // WAITED FOR, NOT SAMPLED (see `RENDER_MS`), and its own act so a + // red says which half went: this one green means the network + // healed and the screen did not. for (const [who, page] of [["A", pageA], ["B", pageB]] as [string, Page][]) { - const rendered = await todoRows(page).allTextContents(); - assert( - rendered.some((t) => t.includes(A_ALONE)) && rendered.some((t) => t.includes(B_ALONE)), - `${who}'s rendered rows after the heal: ${JSON.stringify(rendered)}`, - ); + await until(both, `${who}'s rows to show both cut-off edits`, async () => { + const rendered = await todoRows(page).allTextContents(); + return rendered.some((t) => t.includes(A_ALONE)) && + rendered.some((t) => t.includes(B_ALONE)); + }, RENDER_MS).catch(async (e) => { + throw new Error( + `${e instanceof Error ? e.message : e}\n` + + ` ${who}'s engine holds the merged set but its rows do not show it ` + + `after ${RENDER_MS / 1000}s: ` + + `${JSON.stringify(await todoRows(page).allTextContents())}\n` + + ` ${await diagnoseBoth(pageA, pageB)}`, + ); + }); } }); }, diff --git a/demo/e2e/scenarios/relay-partition.ts b/demo/e2e/scenarios/relay-partition.ts index aa60e20a..259c1757 100644 --- a/demo/e2e/scenarios/relay-partition.ts +++ b/demo/e2e/scenarios/relay-partition.ts @@ -1,14 +1,30 @@ -// A RELAY OUTAGE BETWEEN TWO LIVE SIBLINGS — and the heal that does not -// come. Registered `expected: "red"`: this scenario PINS A GAP. +// A RELAY OUTAGE BETWEEN TWO LIVE SIBLINGS, AND THE HEAL THAT COMES BY +// ITSELF. Green since #113 landed end to end; it was `expected: "red"` +// for three waves of gap before that, and the history is kept below +// because each wave hid the next one. // -// ─── THE CLAIM IT WOULD MAKE IF THE GAP WERE CLOSED ────────────────── +// ─── THE CLAIM ─────────────────────────────────────────────────────── // // Two devices of one account are paired and syncing live. The relay // they meet over goes away; each device is edited while it is alone; // the relay comes back. Both devices then hold the SAME todo set — // merged, not merely one-way delivered — with nobody reloading // anything, nobody re-running a ceremony, and nobody pressing a button. -// That is what a user means by "my other device caught up". +// That is what a user means by "my other device caught up", and every +// step of it now happens on its own: +// +// the relay dies both sides' `conn-status` reports `gone:` +// in ~0.1s (the relay leg is a websocket over +// TCP, so the socket dies under each endpoint +// and it learns synchronously) +// …the outage… each page keeps taking edits locally +// the relay returns +// +~5s both pages rebind their endpoint — same +// address, off the persisted transport key — +// repost the acceptor, and re-dial +// +~5s from restore BOTH engines hold the merged four-todo set +// (measured 4.8–5.4s over five runs) +// +~0.3s and both SCREENS show it (0.26–0.54s) // // ─── WHICH PATH CARRIES THE BYTES (asked first, answered empirically) // @@ -21,87 +37,131 @@ // two reasons that are both in the engine's own source: // // * the endpoint is bound with WebRTC LEFT OFF. `iroh-bind` sets an -// ALPN pair and a relay URL and nothing else (engine/guest/src/lib.rs -// :4088-4096); `endpoint-options.webrtc` is never called, and -// iroh.wit says of it "When disabled (the default), `webrtc` entries -// are ignored for dialing and inbound signaling is discarded"; +// ALPN pair and a relay URL and nothing else; +// `endpoint-options.webrtc` is never called, and iroh.wit says of it +// "When disabled (the default), `webrtc` entries are ignored for +// dialing and inbound signaling is discarded"; // * the dial address offers no other wire anyway — `iroh-start` builds -// `EndpointAddr { addrs: vec![TransportAddr::Relay(relay_url)] }` -// (lib.rs:4134), one relay entry, no `webrtc` and no `ip:port`. +// `EndpointAddr { addrs: vec![TransportAddr::Relay(relay_url)] }`, +// one relay entry, no `webrtc` and no `ip:port`. // // MEASURED, not merely read: with two paired pages converging in 4ms, // `ctx.stopRelay()` and then a todo authored on each side, NOTHING -// crossed in either direction for 60s. The relay is the path. +// crossed in either direction for 60s. The relay is the path — which is +// what makes the outage act below a real partition and this file's +// claim about the relay rather than about localhost. // -// ─── AND THE HEAL DOES NOT HAPPEN. WHERE THE GAP IS ────────────────── +// ─── THE THREE WAVES OF RED, KEPT ──────────────────────────────────── // -// Same probe, continued: `ctx.startRelay()`, then 240s of watching with -// both pages alive and being ticked. Neither side ever saw the other's -// outage edit. Traced, in the page's own terms: +// This file spent its whole life so far as an `expected: "red"` pin, and +// it was red for three DIFFERENT reasons in turn. Each one was only +// findable once the one before it was fixed, which is the argument for +// writing all three down rather than the last. // -// * THE ONLY PATIENT RETRY ON THIS PAGE IS `resumeWire`, and it is -// entered only from the RESUMED-BOOT branch (demo/host/solo.ts:3366 -// and :3374 — the two `void resumeWire(…)` calls, both inside the -// `probe.ok` arm that means "this device already held the account -// when the page loaded"). Its tick loop is the thing that re-dials -// every RESUME_TICK_MS (solo.ts:3016, :3214) and the only caller of -// `rebindEndpoint` for a `Closed` endpoint (solo.ts:2879, :3132). -// * A PAGE THAT PAIRED IN THIS SESSION NEVER ENTERS IT. It is in a -// CEREMONY role — `joinerWire` (solo.ts:2673) or `adderWire` -// (:2900) — each of which runs at most once: the `joinWired` / -// `adderWired` guards latch true on the first success, and the -// `WIRE_ATTEMPTS = 3` budget (solo.ts:2666) is a retry for wiring -// that FAILED TO COME UP, not a re-dial for a wire that came up and -// later died. So when the relay dies under a freshly-paired pair, -// there is no loop left running that would ever dial again. -// * AND NEITHER SIDE CAN EVEN LEARN THE WIRE DIED, which is why the -// absence of a retry has no symptom. `conn-status` reports the -// outcome of the HANDSHAKE and is never invalidated afterwards -// — `iroh-start`'s spawned wiring writes the outcome into -// `conn_results` once and nothing ever removes it -// (engine/guest/src/lib.rs:4407, and :4343/:4400 for the two error -// outcomes), and `conn-status` reads that map back for ever -// (:4413-4420). `sync-status` is one-shot per round rather than a -// subscription's health. solo.ts:2988-3012 writes this down -// already, for the neighbouring one-sided-reload case; the relay -// outage is the same engine limit reached by a different road. -// (solo.ts's committed comment there still cites this latch by its -// OLD line numbers — the code moved, the fact did not.) +// WAVE 1 — THE PAGE COULD NOT RETRY, AND COULD NOT HAVE (issue #113 as +// filed). Two halves, and neither was any use without the other: // -// THE MISSING PIECE, in the engine's own terms: a `conn-status` that -// goes false when the connection drops (solo.ts:3009-3012 names exactly -// this — filed as #113), plus a page-side retry that is armed for the -// life of the page rather than only on the resumed-boot path. With the -// first, the second is cheap and cannot double-dial; without it, any -// re-dial-on-a-timer would be the double-dialling solo.ts's direction -// discipline exists to prevent — which is why this is pinned as a gap -// rather than papered over in a scenario-local workaround. - +// * NO LOOP. A page that paired in THIS session is in a ceremony role +// — `joinerWire` / `adderWire` in demo/host/solo.ts — and both latch +// on their first success. `WIRE_ATTEMPTS = 3` is a retry for wiring +// that never CAME UP, not a re-dial for a wire that came up and +// died. The only patient loop, `resumeWire`, was entered exclusively +// from the resumed-boot branch. So when the relay died under a +// freshly-paired pair there was no loop left running anywhere. +// * AND NO SIGNAL TO LOOP ON. `conn-status` reported the outcome of +// the HANDSHAKE and was never invalidated afterwards: written into +// `conn_results` once, read back for ever. A handle to a peer that +// had been unreachable for an hour answered exactly as a live one +// did. Re-dialling on a timer against that would have been the +// double-dial the direction discipline exists to prevent (#78) — a +// second connection carrying the same subscriptions, silently. +// +// Measured then: no convergence in 240s, both pages alive and ticking +// throughout. +// +// WAVE 2 — THE PAGE HALF, which fixed the above and MOVED the gap into +// the engine. `conn-status` grew a machine-readable `gone:` marker +// (every connection gets a monitor on the iroh WIT's `wait-closed`; +// gated by demo/host/conn-gone-check.ts, ~0.1s on a relay kill), and +// solo.ts grew ONE WIRE-KEEPER armed for the life of every paired page +// rather than only on resumed boots: each 5s tick it reads `conn-status` +// on the handles it holds, and a `gone:` — and nothing else — clears +// that peer so the next tick re-dials. A third fact turned up here and +// is worth keeping: a relay's death does not merely kill CONNECTIONS, +// it latches this device's own ENDPOINT `Closed`, so the keeper's +// repair path runs through `rebindEndpoint` (which re-mints the same +// address off the persisted key) before a dial can land. +// +// Measured then: the transport came ALL the way back — both sides +// rebound, re-dialled, and reported a live connection within ~10s of +// the relay returning — and then not one todo crossed in 150s. The +// reader's `sync-start` returned a handle whose `sync-status` never +// settled; the acceptor's settled fine. +// +// WAVE 3 — THE STALE TRANSPORT CHAIN, which is what that asymmetry was. +// Four links, each individually reasonable: +// +// 1. `QueueTransport` held its OWN channel ends — `in_tx` alongside +// `in_rx` — so `recv_bytes` COULD NOT FAIL. When `iroh_reader` hit +// EOF it dropped only its clone of the sender; the channel stayed +// open and the transport went on politely awaiting frames from a +// socket that no longer existed. +// 2. Subduction's teardown is driven ENTIRELY by a connection's reader +// failing — the per-connection loop's exit is the only thing that +// posts a closure, which is the only thing that removes the +// connection. A transport that cannot fail is therefore never +// removed. +// 3. `add_connection` APPENDS rather than replaces, so after a rebind +// the peer owned a DEAD connection at index 0 and the live one at +// index 1. +// 4. And `sync_with_peer` walks that list IN ORDER, under this +// engine's never-firing timeout — so it called the dead one first +// and parked there for ever, never reaching the live one. +// +// THE ASYMMETRY EXPLAINED: only the side that CALLS `sync_with_peer` +// walks the stale list. An acceptor answering an inbound request +// replies on the connection the message arrived on, so it settled +// normally — which is why `one-sided-reload` was green throughout and +// only the dialling side ever hung. +// +// THE FIX, one link, no upstream change and nothing reaching into +// subduction's registries behind its back: the gone-monitor now CLOSES +// the dead connection's inbound queues, so `recv_bytes` starts +// failing, subduction's own teardown runs, and the next +// `sync_with_peer` finds only the live connection. +// +// Pinned by demo/host/rebind-sync-check.ts (`just rebind-sync`), the +// headless gate written for exactly this and red→green across the fix: +// post-rebind `sync-start` settles in 0.20s and a todo crosses in +// 0.03–0.06s, both directions, where before neither settled at all. // // ─── WHAT IS NOT THE SUBJECT ──────────────────────────────────────── // -// NOTHING RELOADS HERE. The one-sided-reload gap (solo.ts:2988-3012) is -// a different track's; the double reload is `solo-resume-sync`'s claim -// and it is GREEN, precisely because a reloaded page does enter -// `resumeWire`. The distinction is the finding: this account can -// survive both its devices being closed and reopened, but not its relay +// NOTHING RELOADS HERE, and that is the point of the file: a reload +// re-handshakes from nothing, which is the state this scenario exists +// NOT to be in. The double reload is `solo-resume-sync`'s claim and the +// one-sided reload is `one-sided-reload`'s; with this file green, all +// three shapes of "the wire went away" are now covered — both devices +// closed and reopened, one device vanishing, and the relay itself // blinking while both stay open. // // NO STORAGE IS BOUND, deliberately. `solo-offline-sync` shows a todo // crossing through a Drive bucket with no live peer at all; if this -// scenario bound a store, a heal could be the BUCKET's doing and the +// scenario bound a store, the heal could be the BUCKET's doing and the // relay claim would be unfalsifiable. Neither page here has any store, // so the relay is the only channel that exists. // // ─── HOW IT FAILS WHEN IT FAILS ───────────────────────────────────── // -// The red act is the LAST one, and everything before it is a plain -// green precondition — pairing, a live crossing, a real cut. An xfail -// that went red because pairing broke would be a gap flag hiding a -// regression, so the last act's timeout carries a full diagnosis of -// both pages (todos, `usSynced`, sync status, endpoint id) in the -// solo-offline-sync manner, and the acts before it assert their own -// preconditions loudly. +// Everything before the heal is a plain precondition — pairing, a live +// crossing, a real cut — and each asserts itself loudly, so a failure +// there reads as the regression it is rather than as a heal that did not +// come. The heal itself is split into TWO acts on purpose: the engines +// agreeing and the screens following are different claims with different +// failure modes, and keeping them apart means a red says which one went. +// The convergence act's timeout carries a full diagnosis of both pages +// (todos, `usSynced`, sync status, endpoint id, and the wire-keeper's own +// view of its handles) in the solo-offline-sync manner. import type { Page } from "npm:playwright@1.57.0"; import type { Ctx, Scenario } from "../run.ts"; @@ -150,24 +210,48 @@ const CONTROL_MS = 30_000; * return until the port REFUSES.) */ const OUTAGE_WATCH_MS = 30_000; -/** WHAT THE HEAL IS GIVEN, derived rather than guessed — sized to what - * SHOULD work if the missing piece existed, since an xfail whose wait - * was too short would be pinning the harness's impatience instead of - * the gap: +/** WHAT THE HEAL IS GIVEN, sized to the measurement rather than to a + * guess at machinery that did not exist yet: * - * RESUME_TICK_MS 5s solo.ts:3016 — the page's own retry cadence - * + dial deadline 30s solo.ts:2641 — `until` around conn-status - * + subscribe 30s solo.ts:2617 — `until` around sync-status - * + relay pull 45s the relay-mediated pull cadence run.ts's - * `deadlineMs` note names - * ------------------------ - * 110s, and a missed tick costs at most another 5s + * ≤5s the wire-keeper's tick noticing the relay is back and + * running its repair (solo.ts's `KEEPER_TICK_MS`) + * + ~1s rebind, repost, re-dial, re-handshake (measured: the + * post-rebind `sync-start` settles in 0.20s headless, and a + * todo crosses in 0.03–0.06s — demo/host/rebind-sync-check.ts) + * + slack for a missed tick and a busy CI box + * ------ + * MEASURED end to end on this harness: 4.8–5.4s over five + * consecutive runs, from `startRelay()` to both ENGINES holding the + * merged set. * - * Rounded up to 150s: two full dial-and-subscribe attempts plus a pull - * cadence, which is a generous reading of every retry this page could - * plausibly grow. The probe watched 240s and saw nothing move, so the - * red is not this number being tight. */ -const HEAL_MS = 150_000; + * 60s is that with an order of magnitude of margin. It is not sized to + * the old never-happens world — the previous 150s existed to give a heal + * that was not coming every retry it might plausibly have grown, and + * that reasoning is stale. This bound's only job now is to absorb a slow + * box without letting a genuine regression cost the suite minutes. */ +const HEAL_MS = 60_000; + +/** AND WHAT THE SCREENS ARE GIVEN, once the engines agree. + * + * A REPAINT IS NOT A CLAIM VIOLATION, which is the whole reason this is + * a bounded wait and not a bare read. The engine convergence above is + * the network fact; the rows are rendered off a LATER drain, so sampling + * the DOM in the same breath catches a page mid-frame and fails on a row + * that appears milliseconds later. That was the last intermittent red in + * this file's life — a zero-tolerance DOM read racing the repaint, which + * alternated which device's row it found "missing" — and every other row + * assertion in this suite already waits. + * + * 30s: the drain that repaints runs on a 1s cadence, so a healthy paint + * is one or two of those behind the engine. Thirty of them is generous + * enough that a loaded box cannot turn a paint into a red, and short + * enough that a screen which genuinely never updates reports promptly — + * which is a real bug worth failing on, and the reason this is asserted + * at all rather than dropped as "the engine agreed, good enough". + * + * Measured: 0.26–0.54s over five consecutive runs, so the bound carries + * roughly two orders of magnitude of headroom. */ +const RENDER_MS = 30_000; /** Everything a timeout in this scenario should be able to say about a * page, in the shape solo-offline-sync attaches to its own waits: what @@ -186,6 +270,12 @@ async function diagnose(label: string, page: Page): Promise { await read("todos", () => solo(page, "todos")), await read("usSynced", () => solo(page, "usSynced")), await read("sync", () => solo(page, "syncStatus")), + // THE WIRE-KEEPER'S OWN VIEW, added with #113 and the single most + // useful line in this diagnosis now: which handles the page holds and + // what `conn-status` says about each. A reader meeting this failure + // needs to know whether the transport came back — because it does, + // and everything upstream of that fact is already working. + await read("wire", () => solo(page, "wireHealth")), // A PREFIX, not the whole id: 64 hex characters per page would // drown the two facts either side of it, and all a reader needs // from a transport address here is whether it is present and @@ -210,14 +300,13 @@ async function engineTodos(page: Page): Promise { const scenario: Scenario = { name: "relay-partition", why: - "a relay outage between two LIVE paired devices heals by itself when the relay returns — no reload, no ceremony (XFAIL: it does not; see this file's banner)", - // PINNED AS A GAP. Drop this flag the day the last act passes; the - // runner fails an `expected: "red"` scenario that goes green, which - // is what forces that promotion rather than leaving a stale flag. - expected: "red", - // Boot + pairing + CONTROL_MS + OUTAGE_WATCH_MS + HEAL_MS ≈ 250s of - // deliberate waiting, well past the suite-wide 240s. - deadlineMs: 420_000, + "a relay outage between two LIVE paired devices heals by itself when the relay returns — the wire is marked gone, the endpoint rebound, the dial remade and subduction resumed, and both devices converge with no reload and no ceremony", + // Boot + pairing + CONTROL_MS's crossing + the 30s OUTAGE_WATCH_MS + // window + a ~5s heal ≈ 39s measured (38.7–39.2s over five runs). 180s is comfortably over 4x + // that: the two deliberate 30s waits are what this scenario costs, and + // a heal that has regressed reports inside HEAL_MS long before this + // outer deadline is reached. + deadlineMs: 180_000, page: { path: "/solo.html", bootGlobal: "__solo", @@ -389,7 +478,7 @@ const scenario: Scenario = { await input.fill(""); }); - // --- THE CLAIM (this is the act that is red) ------------------------ + // --- THE CLAIM, in two acts ----------------------------------------- await act("the relay returns and BOTH devices converge, with nothing reloaded", async () => { await ctx.startRelay(); @@ -407,26 +496,46 @@ const scenario: Scenario = { ? true : false; }, HEAL_MS).catch(async (e) => { - // THE DIAGNOSIS, because this act is the one a reader will meet - // as a failure — either as the expected red or, one day, as a - // regression in whatever closed the gap. + // THE DIAGNOSIS, because this is the act a reader will meet as a + // regression in whatever closed the gap — and the `wire` field + // says WHICH gap came back. Two LIVE wires with nothing crossing + // is the wave-3 shape (subduction holding a stale connection); + // a dead, missing or never-re-dialled wire is wave 2 (the page's + // wire-keeper); a wire still reading alive on a relay that has + // been dead for half a minute is wave 1 (the `gone:` marker). + // The banner's arithmetic for each is above. throw new Error( `${e instanceof Error ? e.message : e}\n` + - ` the relay came back and the two devices did not re-find each other ` + + ` the relay came back and the two devices did not converge ` + `within ${HEAL_MS / 1000}s. Expected exactly ${JSON.stringify(want)} on both.\n` + ` ${await diagnoseBoth(pageA, pageB)}`, ); }); - // AND ON THE ROWS TOO, once the engines agree: the user's - // complaint is about a screen, and a partition that healed in the - // engine but not on the surface is still a device that "stopped - // updating". + }); + + await act("and both SCREENS follow: each device RENDERS the other's outage edit", async () => { + // THE USER'S OWN TEST. A partition that healed in the engine but + // not on the surface is still a device that "stopped updating", so + // the claim is not finished until the rows say so on both sides. + // + // WAITED FOR, NOT SAMPLED — see `RENDER_MS`. The engines agreed in + // the act above; the rows are painted off a later drain, and a bare + // read here raced that repaint. It is its own act so that a red + // says plainly which half went: this one green means the network + // healed and the surface did not. for (const [who, page] of [["A", pageA], ["B", pageB]] as [string, Page][]) { - const rendered = await todoRows(page).allTextContents(); - assert( - rendered.some((t) => t.includes(A_ALONE)) && rendered.some((t) => t.includes(B_ALONE)), - `${who}'s rendered rows after the heal: ${JSON.stringify(rendered)}`, - ); + await until(both, `${who}'s rows to show both outage edits`, async () => { + const rendered = await todoRows(page).allTextContents(); + return rendered.some((t) => t.includes(A_ALONE)) && + rendered.some((t) => t.includes(B_ALONE)); + }, RENDER_MS).catch(async (e) => { + throw new Error( + `${e instanceof Error ? e.message : e}\n` + + ` ${who}'s engine holds the merged set but its rows do not show it ` + + `after ${RENDER_MS / 1000}s: ${JSON.stringify(await todoRows(page).allTextContents())}\n` + + ` ${await diagnoseBoth(pageA, pageB)}`, + ); + }); } }); }, diff --git a/demo/host/conn-gone-check.ts b/demo/host/conn-gone-check.ts new file mode 100644 index 00000000..fc308337 --- /dev/null +++ b/demo/host/conn-gone-check.ts @@ -0,0 +1,288 @@ +// The gate for #113's ENGINE half: `conn-status` must stop being +// write-once. A wire that came up and later DIED has to say so, in a +// way a page can match on, or no page path may ever re-dial (a second +// dial to a live peer is a second connection carrying the same +// subscriptions — #78's direction discipline — so "retry on a timer" +// is only safe once "this one is dead" is knowable). +// +// deno run -A host/conn-gone-check.ts (or: just conn-gone) +// +// NEEDS NO `just infra`: the probe spawns its OWN iroh-relay child on an +// EPHEMERAL port, because killing the relay is the whole experiment and +// a shared one on :3340 would take everyone else's runs down with it. +// Same reason the port is never fixed: sibling worktrees run this +// concurrently. +// +// The beats: +// 1. two engines, bound to this probe's relay, wired alice→bob; +// conn-status reports the peer on BOTH sides. +// 2. the healthy-path CONTROL: repeated reads over several seconds +// keep reporting Ok. ("Gone" that fires on a live wire would pass +// beat 4 for the wrong reason.) +// 3. SIGKILL the relay. +// 4. poll until both sides report the `gone: ` marker, printing the +// measured kill→gone latency for each side. +// 5. LATCHED: read again, seconds later, still gone. +// +// WHAT GOVERNS THE LATENCY — and the surprise. The theory going in was +// the QUIC IDLE TIMEOUT: nothing writes on an idle connection, so +// nothing learns the path is gone from an error, and the connection +// would only end tens of seconds later when the timeout fired. That is +// NOT what this rig measures. These connections are relay-dialed, and +// the relay leg is a WEBSOCKET over TCP: killing the relay process +// closes that socket, and the endpoint learns synchronously. Measured +// kill→gone here is UNDER A SECOND on both sides. +// +// The idle timeout still exists and still governs the cases where no +// socket dies — a black-holing relay, a NAT drop, a cut path — so +// callers must treat "gone" as eventually-consistent on the order of +// tens of seconds, and `BOUND_MS` is sized for that slower mechanism +// rather than for the fast one this probe happens to trigger. Both +// numbers belong to the pinned endpoint (jsr:@polymorph/iroh@0.3.0), +// not to this engine, which is why this probe MEASURES and PRINTS +// rather than asserting a figure. + +import { type Engine, newEngine, unhex, until } from "../../runtime/engine.ts"; +import { probeNoNet } from "./probe-net.ts"; + +const ENVELOPE = new URL("../build/engine.plan.json", import.meta.url); +const WASM = new URL("../../engine/target/composed.wasm", import.meta.url); +const RELAY_BIN = new URL("../../engine/.deps/relay/bin/iroh-relay", import.meta.url).pathname; + +/// Measured kill→gone on this rig: under a second, both sides (the +/// websocket teardown above). The bound is not measured+50%, and +/// deliberately so: the fast path is an artifact of killing a LOCAL +/// relay process, and a bound tight enough to gate that would turn into +/// a flake the first time this runs somewhere the socket lingers and +/// the idle timeout has to do the work instead. 20s is comfortably +/// above both, and still far below "the gate hung". +const BOUND_MS = 20_000; +/// The contract's machine-readable marker (engine/guest/wit/engine.wit, +/// `conn-status`). The page half matches on this same prefix. +const GONE = "gone:"; + +async function freePort(): Promise { + const l = Deno.listen({ port: 0 }); + const port = (l.addr as Deno.NetAddr).port; + l.close(); + return await Promise.resolve(port); +} + +/** The e2e suite's `Relay` class, trimmed to what one probe needs + * (demo/e2e/run.ts:493). `/generate_204` is the relay's own net-report + * endpoint: answering it is the relay saying it is SERVING, which is + * stronger than the port being open — and refusing it is how we know + * the kill landed rather than merely being issued. */ +class Relay { + #proc: Deno.ChildProcess | null = null; + #dir: string | null = null; + readonly url: string; + constructor(readonly port: number) { + this.url = `http://127.0.0.1:${port}`; + } + + async start(): Promise { + try { + await Deno.stat(RELAY_BIN); + } catch { + console.error( + `no iroh-relay at ${RELAY_BIN} — run \`cd engine && just relay-bin\``, + ); + Deno.exit(2); + } + this.#dir ??= await Deno.makeTempDir({ prefix: "pm-conn-gone-relay." }); + const cfg = `${this.#dir}/relay.toml`; + await Deno.writeTextFile( + cfg, + `http_bind_addr = "127.0.0.1:${this.port}"\nenable_metrics = false\n`, + ); + this.#proc = new Deno.Command(RELAY_BIN, { + args: ["--dev", "--config-path", cfg], + stdout: "null", + stderr: "null", + }).spawn(); + for (let i = 0; i < 120; i++) { + try { + const r = await fetch(`${this.url}/generate_204`, { signal: AbortSignal.timeout(2_000) }); + await r.body?.cancel(); + if (r.status === 204 || r.ok) return; + } catch { /* not up yet */ } + await new Promise((r) => setTimeout(r, 250)); + } + throw new Error(`the local relay never answered on ${this.url}`); + } + + async stop(): Promise { + if (!this.#proc) return; + const proc = this.#proc; + this.#proc = null; + try { + proc.kill("SIGKILL"); + } catch { /* already dead */ } + await proc.status; + for (let i = 0; i < 80; i++) { + try { + const r = await fetch(`${this.url}/generate_204`, { signal: AbortSignal.timeout(2_000) }); + await r.body?.cancel(); + } catch (e) { + // A TIMEOUT is not "down" — only a refusal is (the e2e Relay + // makes the same split for the same reason). + if (e instanceof DOMException && e.name === "TimeoutError") continue; + return; + } + await new Promise((r) => setTimeout(r, 100)); + } + throw new Error("the relay kept answering after being killed"); + } + + async dispose(): Promise { + await this.stop(); + if (this.#dir) await Deno.remove(this.#dir, { recursive: true }).catch(() => {}); + } +} + +/** What a `conn-status` read looks like from TS. The WIT + * `result, string>` lowers to resolve-or-THROW + * (runtime/engine.ts's Driver contract), so an Err — including the + * gone marker — arrives as a thrown exception whose message carries + * the guest's string. Both shapes are folded here so the assertions + * below read as three states, not as try/catch plumbing. */ +type Status = + | { tag: "alive"; peer: string } + | { tag: "gone"; message: string } + | { tag: "failed"; message: string } + | { tag: "unknown" }; + +async function status(e: Engine, conn: number): Promise { + try { + const peer = await e.driver.connStatus(conn); + return peer === undefined ? { tag: "unknown" } : { tag: "alive", peer }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return message.includes(GONE) ? { tag: "gone", message } : { tag: "failed", message }; + } +} + +function dumpOnFail(engines: [string, Engine][]) { + for (const [name, e] of engines) { + const err = e.stderr(); + if (err.trim()) console.error(`--- ${name} stderr ---\n${err}`); + } +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +async function main() { + const artifacts = { + envelope: await Deno.readTextFile(ENVELOPE), + bytes: await Deno.readFile(WASM), + }; + const relay = new Relay(await freePort()); + await relay.start(); + console.log(`relay up on ${relay.url} (ephemeral port, this probe's own child)`); + + const alice = await newEngine("alice", artifacts, probeNoNet); + const bob = await newEngine("bob", artifacts, probeNoNet); + try { + await alice.driver.init(false); + const bobId = unhex(await bob.driver.init(false)); + await alice.driver.irohBind(relay.url); + const bobEp = unhex(await bob.driver.irohBind(relay.url)); + + // Acceptor first, then the dial — the acceptor's `accept()` must be + // pending before anyone connects (host/bringup.ts's wire phase does + // exactly this ordering). + const cb = await bob.driver.irohStart(false, new Uint8Array(), relay.url, new Uint8Array()); + const ca = await alice.driver.irohStart(true, bobEp, relay.url, bobId); + await until( + "subduction handshake over the probe's relay", + async () => (await alice.driver.connStatus(ca)) && (await bob.driver.connStatus(cb)), + ); + console.log("both sides report a peer: the wire is up"); + + // --- the healthy-path control ------------------------------------- + // + // The monitor must not fire on a LIVE connection. Without this, beat + // 4 would also pass for an engine that simply marked everything gone + // after a few seconds. + for (let i = 0; i < 6; i++) { + await sleep(1_000); + for (const [who, e, c] of [["alice", alice, ca], ["bob", bob, cb]] as const) { + const s = await status(e, c); + if (s.tag !== "alive") { + throw new Error(`CONTROL FAILED: ${who} is ${s.tag} while the relay is up ` + + `(${"message" in s ? s.message : "-"})`); + } + } + } + console.log("control: 6s of repeated reads, both sides stayed alive"); + + // --- the kill ----------------------------------------------------- + // + // `t0` is the instant the SIGKILL is ISSUED, not the instant + // `relay.stop()` returns — stop() additionally waits for the port to + // refuse, and folding that wait into the latency would flatter the + // measurement. + const t0 = performance.now(); + const stopped = relay.stop(); + console.log("relay SIGKILLed — waiting for the wire to notice"); + + const seen: Record = {}; + const messages: Record = {}; + while (performance.now() - t0 < BOUND_MS) { + for (const [who, e, c] of [["alice", alice, ca], ["bob", bob, cb]] as const) { + if (seen[who] !== undefined) continue; + const s = await status(e, c); + if (s.tag === "gone") { + seen[who] = performance.now() - t0; + messages[who] = s.message; + console.log(` ${who}: gone after ${(seen[who] / 1000).toFixed(2)}s — ${s.message}`); + } else if (s.tag === "failed") { + throw new Error(`${who} reports a NON-gone error after the kill: ${s.message} ` + + `(the gone marker is the contract; a bare error is not it)`); + } + } + if (seen.alice !== undefined && seen.bob !== undefined) break; + await sleep(100); + } + for (const who of ["alice", "bob"]) { + if (seen[who] === undefined) { + throw new Error( + `${who} never reported "${GONE}" within ${BOUND_MS / 1000}s of the relay dying — ` + + `conn-status is still write-once on that side`, + ); + } + } + await stopped; + console.log("relay confirmed refusing (SIGKILL landed, not merely issued)"); + + // --- latched ------------------------------------------------------ + // + // `wait-closed` is latched at the iroh layer and the engine writes + // the marker once; the point of reading again is that nothing + // LATER (a reconnect attempt, a handshake settling late) walks the + // entry back to Ok. + await sleep(5_000); + for (const [who, e, c] of [["alice", alice, ca], ["bob", bob, cb]] as const) { + const s = await status(e, c); + if (s.tag !== "gone") throw new Error(`${who} un-latched: re-read says ${s.tag}`); + } + console.log("latched: 5s later both sides still report gone"); + + console.log( + `\nMEASURED kill→gone: alice ${(seen.alice / 1000).toFixed(1)}s, ` + + `bob ${(seen.bob / 1000).toFixed(2)}s (bound ${BOUND_MS / 1000}s).`, + ); + console.log("conn-gone-check: OK"); + } catch (e) { + dumpOnFail([["alice", alice], ["bob", bob]]); + throw e; + } finally { + await relay.dispose(); + } +} + +if (import.meta.main) { + await main(); + Deno.exit(0); +} diff --git a/demo/host/rebind-sync-check.ts b/demo/host/rebind-sync-check.ts new file mode 100644 index 00000000..60b3d075 --- /dev/null +++ b/demo/host/rebind-sync-check.ts @@ -0,0 +1,352 @@ +// The gate for #113's RESIDUAL gap: subduction sync must survive an +// ENDPOINT REBIND, not just a fresh connection. +// +// deno run -A host/rebind-sync-check.ts (or: just rebind-sync) +// +// WHY A SECOND PROBE. `conn-gone-check.ts` proves the engine NOTICES a +// dead wire; this one proves the engine can USE the replacement. Those +// are different claims and the second one was false: after a relay +// outage both pages rebind their endpoint, re-dial, and `conn-status` +// goes LIVE within seconds — and then nothing crosses, for minutes, +// because `sync-start` on the reader side never settles. A liveness +// signal that leads to a wire nobody can sync over is a worse bug than +// no signal, so it gets its own gate. +// +// Same rig as conn-gone-check: this probe spawns its OWN relay child on +// an EPHEMERAL port (killing it is the experiment; a shared :3340 would +// take every other run down with it, and a fixed port collides with +// sibling worktrees). +// +// THE BEATS: +// 1. two engines, wired, a partition created/sealed/adopted, both +// subscribed — and a todo actually CROSSES. That is the control: +// the machinery works before anything is broken. +// 2. SIGKILL the relay; wait for both sides to report `gone:` +// (#113's engine half, already gated by conn-gone-check). +// 3. restart the relay on the SAME port, and both sides `iroh-bind` +// AGAIN — which is what the page's `rebindEndpoint` does. A rebind +// keeps the endpoint IDENTITY (the persisted transport key) so the +// peers can still find each other; what it replaces is the +// endpoint RESOURCE and every connection hanging off it. +// 4. re-dial / re-accept; `conn-status` goes live again. +// 5. THE ASSERTION: `sync-start` settles on BOTH sides, and a todo +// written after the outage crosses BOTH ways. +// +// Beat 5 is the one that was red. See the recipe comment in +// demo/justfile for what the failure looked like. + +import { type Engine, hex, newEngine, unhex, until } from "../../runtime/engine.ts"; +import { probeNoNet } from "./probe-net.ts"; + +const ENVELOPE = new URL("../build/engine.plan.json", import.meta.url); +const WASM = new URL("../../engine/target/composed.wasm", import.meta.url); +const RELAY_BIN = new URL("../../engine/.deps/relay/bin/iroh-relay", import.meta.url).pathname; + +/// How long a post-rebind `sync-start` gets to settle before the probe +/// calls it stuck. The healthy path settles in well under a second; the +/// broken path never settles at all, so this is a "how long to wait +/// before believing never" figure, not a performance budget. +const SETTLE_MS = 30_000; +/// How long the todo written after the outage gets to cross. +const CROSS_MS = 45_000; +/// The `conn-status` gone marker (engine.wit's `conn-status` contract). +const GONE = "gone:"; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +async function freePort(): Promise { + const l = Deno.listen({ port: 0 }); + const port = (l.addr as Deno.NetAddr).port; + l.close(); + return await Promise.resolve(port); +} + +/** conn-gone-check.ts's Relay, verbatim in behaviour: `/generate_204` + * is the relay's own net-report endpoint, so answering it means SERVING + * and refusing it means the kill actually landed. `start()` is + * re-enterable and keeps the SAME config file — same port — which is + * what beat 3 needs: a peer that rebinds must find a relay listening + * where it expects one. */ +class Relay { + #proc: Deno.ChildProcess | null = null; + #dir: string | null = null; + readonly url: string; + constructor(readonly port: number) { + this.url = `http://127.0.0.1:${port}`; + } + + async start(): Promise { + if (this.#proc) return; + try { + await Deno.stat(RELAY_BIN); + } catch { + console.error(`no iroh-relay at ${RELAY_BIN} — run \`cd engine && just relay-bin\``); + Deno.exit(2); + } + this.#dir ??= await Deno.makeTempDir({ prefix: "pm-rebind-relay." }); + const cfg = `${this.#dir}/relay.toml`; + await Deno.writeTextFile( + cfg, + `http_bind_addr = "127.0.0.1:${this.port}"\nenable_metrics = false\n`, + ); + this.#proc = new Deno.Command(RELAY_BIN, { + args: ["--dev", "--config-path", cfg], + stdout: "null", + stderr: "null", + }).spawn(); + for (let i = 0; i < 120; i++) { + try { + const r = await fetch(`${this.url}/generate_204`, { signal: AbortSignal.timeout(2_000) }); + await r.body?.cancel(); + if (r.status === 204 || r.ok) return; + } catch { /* not up yet */ } + await sleep(250); + } + throw new Error(`the local relay never answered on ${this.url}`); + } + + async stop(): Promise { + if (!this.#proc) return; + const proc = this.#proc; + this.#proc = null; + try { + proc.kill("SIGKILL"); + } catch { /* already dead */ } + await proc.status; + for (let i = 0; i < 80; i++) { + try { + const r = await fetch(`${this.url}/generate_204`, { signal: AbortSignal.timeout(2_000) }); + await r.body?.cancel(); + } catch (e) { + // A TIMEOUT is not "down"; only a refusal is. + if (e instanceof DOMException && e.name === "TimeoutError") continue; + return; + } + await sleep(100); + } + throw new Error("the relay kept answering after being killed"); + } + + async dispose(): Promise { + await this.stop(); + if (this.#dir) await Deno.remove(this.#dir, { recursive: true }).catch(() => {}); + } +} + +/** `conn-status` folded into three states. The WIT + * `result, string>` lowers to resolve-or-THROW, and the + * host prefixes the guest's message ("component error: gone: …"), so + * the marker is matched with `includes`, never `startsWith`. */ +type Status = + | { tag: "alive"; peer: string } + | { tag: "gone"; message: string } + | { tag: "failed"; message: string } + | { tag: "unknown" }; + +async function status(e: Engine, conn: number): Promise { + try { + const peer = await e.driver.connStatus(conn); + return peer === undefined ? { tag: "unknown" } : { tag: "alive", peer }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return message.includes(GONE) ? { tag: "gone", message } : { tag: "failed", message }; + } +} + +/** A `sync-start` that is allowed to NOT settle. `until` throws on + * timeout, which is the right shape for a control and the wrong shape + * for the assertion under test: this returns the summary or `null`, so + * the probe can say WHICH side stuck rather than just failing. */ +async function settle( + who: string, + e: Engine, + peer: Uint8Array, + part: Uint8Array, + subscribe: boolean, + boundMs: number, +): Promise<{ ms: number; summary: string } | null> { + const t0 = performance.now(); + const h = await e.driver.syncStart(peer, part, subscribe); + while (performance.now() - t0 < boundMs) { + try { + const s = await e.driver.syncStatus(h); + if (s !== undefined) { + const ms = performance.now() - t0; + console.log(` ${who} sync settled in ${(ms / 1000).toFixed(2)}s: ${s}`); + return { ms, summary: s }; + } + } catch (err) { + // An ERRORED sync is still a settled sync — and a far better + // outcome than silence, because it names something. + const message = err instanceof Error ? err.message : String(err); + console.log(` ${who} sync settled ERRORED: ${message}`); + return { ms: performance.now() - t0, summary: `ERR ${message}` }; + } + await sleep(200); + } + console.log(` ${who} sync NEVER SETTLED within ${boundMs / 1000}s`); + return null; +} + +function dumpOnFail(engines: [string, Engine][]) { + for (const [name, e] of engines) { + const err = e.stderr(); + if (err.trim()) console.error(`--- ${name} stderr (last 4000) ---\n${err.slice(-4000)}`); + } +} + +async function main() { + const artifacts = { + envelope: await Deno.readTextFile(ENVELOPE), + bytes: await Deno.readFile(WASM), + }; + const relay = new Relay(await freePort()); + await relay.start(); + console.log(`relay up on ${relay.url} (ephemeral port, this probe's own child)`); + + const alice = await newEngine("alice", artifacts, probeNoNet); + const bob = await newEngine("bob", artifacts, probeNoNet); + try { + const aliceId = unhex(await alice.driver.init(false)); + const bobId = unhex(await bob.driver.init(false)); + + // --- beat 1: the CONTROL — a working wire, and data crossing ------ + const aliceEp1 = await alice.driver.irohBind(relay.url); + const bobEp1 = await bob.driver.irohBind(relay.url); + let cb = await bob.driver.irohStart(false, new Uint8Array(), relay.url, new Uint8Array()); + let ca = await alice.driver.irohStart(true, unhex(bobEp1), relay.url, bobId); + await until( + "handshake", + async () => (await alice.driver.connStatus(ca)) && (await bob.driver.connStatus(cb)), + ); + + // Contact cards cross the bridge on their own once the handshake + // lands; `kh-add-member` needs bob's card to exist here first. + await until( + "contact cards over the bridge", + async () => + (await alice.driver.khKnowsAgent(bobId)) && (await bob.driver.khKnowsAgent(aliceId)), + ); + + const part = await alice.driver.createPartition(); + await alice.driver.khAddMember(part, bobId, "edit"); + await alice.driver.sealPartition(part); + await bob.driver.adoptPartition(part); + await until("bob's keyhive knows the doc", () => bob.driver.khKnowsAgent(part)); + + // Pull before subscribe: a subscribe=true FIRST sync reports commits + // received but does not store them (bringup.ts records the same + // ordering constraint). ONE sync-start, then poll ITS handle — + // restarting a sync every poll would be a request storm, not a wait. + const pullHandle = await bob.driver.syncStart(aliceId, part, false); + await until("bob pull", () => bob.driver.syncStatus(pullHandle)); + await until("bob decrypts creation", async () => (await bob.tasks.revision()) >= 1n); + for (const [who, e, peer] of [["bob", bob, aliceId], ["alice", alice, bobId]] as const) { + const h = await e.driver.syncStart(peer, part, true); + await until(`${who} subscribe`, () => e.driver.syncStatus(h)); + } + await alice.tasks.add("before the outage"); + await until("CONTROL: bob sees alice's todo over the live wire", async () => + (await bob.tasks.items()).items.some((i) => i.title === "before the outage")); + console.log("control: the wire works and a todo crossed"); + + // --- beat 2: the outage ------------------------------------------- + const t0 = performance.now(); + await relay.stop(); + console.log("relay SIGKILLed"); + for (const [who, e, c] of [["alice", alice, ca], ["bob", bob, cb]] as const) { + await until(`${who} reports gone`, async () => (await status(e, c)).tag === "gone"); + } + console.log(`both sides gone after ${((performance.now() - t0) / 1000).toFixed(2)}s`); + + // --- beat 3: the relay returns and both sides REBIND --------------- + await relay.start(); + console.log("relay back up on the same port"); + const tRebind = performance.now(); + const aliceEp2 = await alice.driver.irohBind(relay.url); + const bobEp2 = await bob.driver.irohBind(relay.url); + // The endpoint IDENTITY must survive: peers address each other by + // it, and a rebind that minted a fresh one would make the two pages + // undiscoverable to each other for reasons unrelated to sync. (This + // probe's engines have no persisted transport key, so a CHANGED id + // here is expected and is exactly why the re-dial below uses the + // fresh `bobEp2` rather than the stale `bobEp1`.) + console.log( + `rebound: alice ${aliceEp1 === aliceEp2 ? "same" : "new"} endpoint id, ` + + `bob ${bobEp1 === bobEp2 ? "same" : "new"}`, + ); + + // --- beat 4: re-dial ---------------------------------------------- + cb = await bob.driver.irohStart(false, new Uint8Array(), relay.url, new Uint8Array()); + ca = await alice.driver.irohStart(true, unhex(bobEp2), relay.url, bobId); + await until( + "re-handshake after the rebind", + async () => (await alice.driver.connStatus(ca)) && (await bob.driver.connStatus(cb)), + ); + console.log( + `conn-status LIVE again ${((performance.now() - tRebind) / 1000).toFixed(2)}s after the rebind`, + ); + + // --- beat 5: THE ASSERTION ---------------------------------------- + // + // Both directions, because the failure was ASYMMETRIC: the side that + // accepted settled and the side that dialled did not, and a probe + // that only checked one of them would have been green through the + // whole bug. + const aliceSync = await settle("alice", alice, bobId, part, true, SETTLE_MS); + const bobSync = await settle("bob", bob, aliceId, part, true, SETTLE_MS); + const stuck = [ + ...(aliceSync ? [] : ["alice"]), + ...(bobSync ? [] : ["bob"]), + ]; + if (!aliceSync || !bobSync) { + throw new Error( + `sync-start never settled after the rebind on: ${stuck.join(", ")} — ` + + `the connection is LIVE by conn-status but sync cannot use it ` + + `(the stale-registration gap; see this file's banner)`, + ); + } + + // Settling is necessary, not sufficient: a sync can report success + // having talked to nobody. Only data crossing proves the wire. + await alice.tasks.add("after the rebind"); + const tCross = performance.now(); + await until( + "alice's post-rebind todo reaches bob", + async () => (await bob.tasks.items()).items.some((i) => i.title === "after the rebind"), + CROSS_MS, + ); + const aliceToBob = performance.now() - tCross; + + await bob.tasks.add("bob's reply after the rebind"); + const tBack = performance.now(); + await until( + "bob's post-rebind todo reaches alice", + async () => + (await alice.tasks.items()).items.some((i) => i.title === "bob's reply after the rebind"), + CROSS_MS, + ); + const bobToAlice = performance.now() - tBack; + + console.log( + `\nHEALED: alice→bob ${(aliceToBob / 1000).toFixed(2)}s, ` + + `bob→alice ${(bobToAlice / 1000).toFixed(2)}s after the rebind ` + + `(sync settle: alice ${(aliceSync.ms / 1000).toFixed(2)}s, ` + + `bob ${(bobSync.ms / 1000).toFixed(2)}s)`, + ); + console.log("rebind-sync-check: OK"); + // Keeps the unused-binding honest: the ids are read above only to + // report whether the rebind preserved them. + void hex(aliceId); + } catch (e) { + dumpOnFail([["alice", alice], ["bob", bob]]); + throw e; + } finally { + await relay.dispose(); + } +} + +if (import.meta.main) { + await main(); + Deno.exit(0); +} diff --git a/demo/host/solo.ts b/demo/host/solo.ts index 676d6b62..9071eb8c 100644 --- a/demo/host/solo.ts +++ b/demo/host/solo.ts @@ -3981,12 +3981,19 @@ async function startApp( * `us-*` surface, and the engine subscribes the us doc to every known * peer itself on every pump (usdoc.rs's `ensure_subscriptions`: * "Engine-driven because `us-*` hides doc identity by design"). By - * resume time the peer is known, so the engine does it. */ + * resume time the peer is known, so the engine does it. + * + * IT RETURNS THE CONNECTION ID, and #113 is why. A conn id is the only + * handle anything on this page has for asking "is that wire still + * alive?" — `conn-status` and its `gone:` marker, which the wire-keeper + * below reads every tick. Dropping the id on the floor, which is what + * this function used to do, is precisely how a dead wire stayed + * indistinguishable from a healthy one on this side of the boundary. */ const dialPeer = async ( peer: Uint8Array, peerEndpoint: Uint8Array, usPartition?: Uint8Array, - ) => { + ): Promise => { // #115: same shape as `subscribe` above and for the same reason — // each driver call is its own `enqueue` slot, and the 30s // `until` wait for the dial to land runs OUTSIDE any single slot. @@ -3997,6 +4004,7 @@ async function startApp( const conn2 = await enqueue(() => driver.irohStart(true, peerEndpoint, RELAY, peer)); await until("the other device answers", () => enqueue(() => driver.connStatus(conn2)), 30_000); if (usPartition) await subscribe(peer, usPartition, "your account"); + return conn2; }; /** The account's todo list, as the account's own pointer map names it @@ -4015,7 +4023,184 @@ async function startApp( return pointer.id; }; - // --- role: the JOINER (this page is the new device) ---------------------- + // --- the wire-keeper's state, shared by every role (#113) ---------------- + // + // WHY IT LIVES OUT HERE rather than inside the loop that reads it. The + // three wiring roles below are not three machines: they are three ways + // of ARRIVING at the same steady state — this device holding one + // connection per peer, in the direction the ceremony fixed. The + // ceremonies know things the account's directory may not carry yet (an + // enrollment's peer ids, seconds old); the keeper knows how to notice a + // wire die and put it back. So the ceremonies do the first wire-up and + // then HAND THE KEEPER THEIR STATE, and from that moment there is + // exactly one thing on this page responsible for the wire. + // + // Before #113 this state was private to `resumeWire`, because + // `resumeWire` was the only caller that could act on it: nothing could + // tell a live connection from a dead one, so a ceremony-paired page had + // nothing useful to hand over and no safe reason to re-dial. The engine + // half of #113 changed that fact, and this block is the shape the page + // takes once it is true. + + /** THE ONE OUTBOUND CONNECTION this page holds, or null when it holds + * none. `peerHex` is who is on the other end; `conn` is the handle the + * health sweep asks about. + * + * SINGULAR ON PURPOSE, and it is the direction discipline (#78) rather + * than a simplification: a device dials its ENROLLER and nobody else. + * Everyone who dialled THIS device arrives on the acceptor. */ + let dialledPeer: { peerHex: string; conn: number } | null = null; + + /** Which peers this device has already subscribed the todo list to, + * keyed by agent id, hex. Emptied per-peer when that peer's wire dies: + * a subscription is a property OF a connection, so it does not survive + * one, and re-subscribing on a fresh conn is not a double-subscription + * of anything. */ + const tasksWired = new Set(); + + /** The agent ids (hex, lower) of this device's CHILDREN as the last + * directory read saw them. Held out here for one reason: the health + * sweep runs at the TOP of a tick, before that tick has read the + * directory, and when the acceptor dies it needs to know whose + * subscriptions died with it. Last tick's answer is the right one — the + * roster changes at enrollment time, not while a wire is dropping. */ + let childrenHex: string[] = []; + + /** Whether the tasks partition is this device's to read. False only on + * the resume path that boots without a todo-list pointer, until the + * pointer arrives and the adoption runs. */ + let tasksHeld = true; + + /** WHAT ONE CONNECTION HAS TO SAY FOR ITSELF — the whole point of + * #113's engine half, read at the one gate that matters. + * + * CONTRACT (engine/guest/wit/engine.wit, `conn-status`): `Ok(some)` is + * "the handshake succeeded and this is the last known live peer"; + * `Ok(none)` is a handshake still settling; an `Err` whose text carries + * the `gone:` marker is "the wire came up and later DIED", latched; any + * other `Err` is a handshake that failed. + * + * MATCHED WITH `includes`, NOT `startsWith`. The marker is the guest's, + * and by the time it has crossed the component boundary the host has + * prefixed it ("component error: gone: …"). Anchoring at the start + * would be a check that passes in Rust and fails here. + * + * THE FOUR CASES ARE KEPT APART because the keeper does something + * different with three of them: `gone` re-dials, `failed` may be this + * device's own endpoint having died (`Closed`, which is what drives the + * rebind), and `settling` and `alive` are both "leave it alone". A + * two-way boolean would have to fold `failed` into one of the others, + * and folding it into "dead" is the version that re-dials on evidence + * that is not evidence — the double-dial the direction discipline + * exists to prevent. */ + type ConnHealth = + | { kind: "alive"; peer: string } + | { kind: "settling" } + | { kind: "gone"; message: string } + | { kind: "failed"; error: unknown; message: string }; + + const probeConn = async (c: number): Promise => { + try { + // ITS OWN `enqueue` SLOT, and nothing else in it (#115): this is a + // millisecond-sized read — the engine answers out of a map, without + // touching the transport — so the sweep can never be the thing that + // stalls the chain, even with every wire on this page dead. + const peer = await enqueue(() => driver.connStatus(c)); + return peer ? { kind: "alive", peer } : { kind: "settling" }; + } catch (e) { + const message = err(e); + return message.includes("gone:") + ? { kind: "gone", message } + : { kind: "failed", error: e, message }; + } + }; + + /** THE ONE PLACE A DIAL IS ATTEMPTED, and the reason it exists is that + * after #113 there are TWO callers: the ceremony that wires this pair + * for the first time, and the keeper that keeps the wire up afterwards + * — and for a while they overlap. A ceremony whose wiring failed + * mid-flight is still retrying (`WIRE_ATTEMPTS`) while the keeper is + * already ticking, and two dials to one peer is exactly the second + * connection carrying the same subscriptions that #78 forbids. + * + * SINGLE-FLIGHT, TWO GATES, both needed: `dialledPeer` rules out a + * peer already reached, and `dialInFlight` rules out a peer being + * reached RIGHT NOW — a dial has a 30s deadline inside it and this page + * is perfectly capable of ticking six times while one is in the air. + * Returns whether this call is the one that dialled, so a caller with + * follow-up work knows whether the wire is its to finish. + * + * NOT `enqueue`d as a whole (#115): `dialPeer` enqueues each driver + * call individually and waits outside the slots, and this guard is a + * pair of booleans rather than a queue for the same reason — a dial in + * progress must not be able to hold the chain. */ + let dialInFlight = false; + const dialOnce = async ( + peerHex: string, + peer: Uint8Array, + peerEndpoint: Uint8Array, + usPartition?: Uint8Array, + ): Promise => { + if (dialledPeer !== null || dialInFlight) return false; + dialInFlight = true; + try { + const c = await dialPeer(peer, peerEndpoint, usPartition); + dialledPeer = { peerHex, conn: c }; + return true; + } finally { + dialInFlight = false; + } + }; + + /** Subscribe the account's todo list to one peer, at most once per + * live wire. THE SET IS MARKED BEFORE THE AWAIT, which is the same + * discipline the ceremony guards use and for the same reason: a second + * caller arriving while the first is still in `sync-start` would + * otherwise start a second subduction for one pair. It is unmarked + * again if the subscription did not take, or this device would be + * silently unsubscribed for ever. */ + const claimTasks = async (peerHex: string, peer: Uint8Array, tree: Uint8Array) => { + if (tasksWired.has(peerHex)) return; + tasksWired.add(peerHex); + try { + await subscribe(peer, tree, "your todo list"); + } catch (e) { + tasksWired.delete(peerHex); + throw e; + } + }; + + /** Forget the outbound wire: the handle is dead, so the subscriptions + * riding on it are too. Idempotent — the rebind path and the health + * sweep both call it, and they can both be right at once. */ + const forgetDial = () => { + if (!dialledPeer) return; + tasksWired.delete(dialledPeer.peerHex); + dialledPeer = null; + }; + + /** Forget the acceptor, and every subscription that was riding on it. + * + * THE SUBSCRIPTIONS ARE THE SUBTLE HALF, and leaving them behind is a + * bug that looks like nothing: a subduction is a property of the + * CONNECTION it was started on, so when the acceptor is replaced — by a + * repost after a death, or by a rebind taking the whole endpoint with + * it — every child's subscription died with the old handle while this + * page's `tasksWired` still claimed them all as done. The wire then + * comes back up, both sides report a healthy connection, and nothing + * crosses: #78's silence, reached by a fourth road. (Measured: a relay + * bounce healed the transport in ~10s and then sat there for two + * minutes with the acceptor "alive" and not one byte moving, purely + * because this set had not been emptied.) + * + * `childrenHex` rather than the whole set on purpose: the enroller's + * subscription rides the DIAL, and it is `forgetDial`'s to clear. */ + const forgetAcceptor = () => { + acceptorPosted = false; + acceptorConn = null; + for (const child of childrenHex) tasksWired.delete(child); + }; + let joinWired = false; let joinAttempts = 0; @@ -4048,16 +4233,19 @@ async function startApp( throw new Error("the enrollment carried no peer ids — cannot reach the other device"); } const peer = enrollment.peerAgentId; + const peerHex = hex(peer).toLowerCase(); // READER DIALS — and the account doc goes with the dial, because - // the two sides have only just met (see `dialPeer`). - await dialPeer(peer, enrollment.peerEndpointId, enrollment.partitionId); + // the two sides have only just met (see `dialPeer`). Through + // `dialOnce`, so that a retry of this ceremony running alongside + // the keeper cannot make two connections out of one pair. + await dialOnce(peerHex, peer, enrollment.peerEndpointId, enrollment.partitionId); const tasksId = await awaitTasksPointer("your account's todo list", 60_000); // #115: NOT wrapped in one outer `enqueue` any more — `subscribe` // now enqueues its own driver calls (see its definition), and // nesting an `enqueue` inside a job it starts is the self-deadlock // this file's own note on `enqueue` warns about. await enqueue(() => driver.adoptPartition(tasksId)); - await subscribe(peer, tasksId, "your todo list"); + await claimTasks(peerHex, peer, tasksId); usSynced = true; console.log("[solo] subduction wired: this device ⇄ the device that added it"); // THE ADOPTION BEAT, at the only honest moment for it. The join @@ -4092,6 +4280,22 @@ async function startApp( if (joinAttempts < WIRE_ATTEMPTS) joinWired = false; else announce(`could not sync this device with your account: ${err(e)}`, true); console.warn(`[solo] post-enrollment wiring failed (attempt ${joinAttempts}): ${err(e)}`); + } finally { + // #113: AND NOW THE KEEPER TAKES OVER — in a `finally`, which is + // the whole point. A ceremony that FAILED is precisely the page + // that most needs a patient loop: the first thing that ever went + // wrong here was a relay dying mid-`subscribe`, which threw out of + // the try above and left this device with no wire, no retry + // running, and no symptom. Arming the keeper only on the success + // path would have rebuilt that hole one level up. + // + // SAFE ALONGSIDE THE CEREMONY'S OWN RETRIES, because the two now + // share their state: `dialOnce` and `claimTasks` are the only ways + // either reaches the wire, and both are single-flight. The ceremony + // keeps its bounded three attempts (it has the enrollment's ids, + // which the directory may not carry yet); the keeper runs + // underneath it for as long as the page lives. + void wireKeeper(false); } }; @@ -4240,9 +4444,17 @@ async function startApp( console.warn(`[solo] endpoint was closed; rebound the same address ${hex(id).slice(0, 8)}…`); // The old acceptor went down with the endpoint, so the flag has to // as well — otherwise this device would look like it was listening - // while nothing was. - acceptorPosted = false; - acceptorConn = null; + // while nothing was. `forgetAcceptor` additionally drops the + // subscriptions that were riding on it (see its note): they died with + // the handle, and a page that still believes in them never rebuilds + // them. + forgetAcceptor(); + // AND SO DID THE OUTBOUND WIRE (#113). Every connection this page + // held belonged to the endpoint that just went away; a `dialledPeer` + // surviving a rebind would be the keeper guarding a handle minted + // against a transport that no longer exists, which is the same + // stale-handle shape #113 is about, reached from the other side. + forgetDial(); await postAcceptor(); }; @@ -4298,37 +4510,72 @@ async function startApp( // (usdoc.rs's `ensure_subscriptions`, which runs on every pump). // What the engine cannot do for us is the TASKS partition — it // has no name for it — so that one is subscribed here. - if (tasksPart) await subscribe(peer, tasksPart.id, "your todo list"); + if (tasksPart) await claimTasks(joiner.agentId.toLowerCase(), peer, tasksPart.id); usSynced = true; console.log("[solo] subduction wired: this device ⇄ the device it added"); } catch (e) { if (adderAttempts < WIRE_ATTEMPTS) adderWired = false; else announce(`could not sync the new device with your account: ${err(e)}`, true); console.warn(`[solo] post-grant wiring failed (attempt ${adderAttempts}): ${err(e)}`); + } finally { + // #113, in a `finally` for the reason joinerWire's note gives: the + // ceremony that failed is the page that most needs the keeper. On + // this side the state it inherits is `acceptorConn` and whatever + // `claimTasks` managed to record; what it adds is a sweep that + // notices the acceptor die and reposts it. + void wireKeeper(false); } }; - // --- role: a RESUMED BOOT (the account is already here) ------------------ + // --- THE WIRE-KEEPER: this page's standing machinery for the wire --------- // - // THE GAP THIS CLOSES, stated plainly: the two roles above are - // CEREMONY roles. They run once, on the edge of an enrollment, and - // everything they know — who the peer is, where to dial it — came out - // of a ceremony that is over. So a page that reloads afterwards wires - // nothing, and once BOTH devices have reloaded there is no connection - // between them at all: edits stop crossing, and neither page has any - // symptom to show for it. Silence again, which is the failure mode - // this whole file keeps writing down. + // WHAT IT IS. One patient loop, entered ONCE per page and then running + // for the life of it, that keeps this device's connections to its + // account's other devices up — re-dialling, re-accepting and + // re-subscribing whenever the wire it is responsible for dies. // - // WHAT REPLACES THE CEREMONY'S KNOWLEDGE is the account's own device - // directory. Each entry now carries an ENDPOINT (where that device - // can be dialled) and an ENROLLED-BY (which device let it in), so the - // enrollment tree survives in the one place both devices already - // agree about — and a resumed boot can read its own role out of it - // instead of remembering one. + // WHERE IT IS ENTERED FROM, all three of them: + // * a RESUMED BOOT — this device already held the account when the + // page loaded, so there is no ceremony and the directory is the + // only thing that knows who to reach; + // * the JOINER's ceremony, once it has wired; + // * the ADDER's ceremony, once it has wired. + // The ceremonies still do the FIRST wire-up, and it has to be that way: + // an enrollment carries peer ids that are seconds old, while the + // account's directory carries them only once the us doc has synced. + // What changes at #113 is that they hand over rather than finish. // - // THE ROLE IS THE SAME ROLE, and it must be: reversing the direction - // is the healthy-looking silence #78 is about. So this side re-enacts - // exactly what it did at ceremony time — + // THE GAP THIS CLOSES, first half — CEREMONY PAGES NEVER LOOPED. The + // two roles above run at most once each: `joinWired` / `adderWired` + // latch on the first success, and `WIRE_ATTEMPTS = 3` is a retry for + // wiring that never CAME UP, not a re-dial for a wire that came up and + // later died. So a pair that paired in this session and then lost its + // relay had no loop left running anywhere, and stayed dead until + // somebody reloaded a page. (Measured before the fix: no convergence in + // 240s with both pages alive and ticking.) Now both ceremonies end + // here, and the loop is a property of being PAIRED rather than of + // having RESUMED. + // + // THE GAP THIS CLOSES, second half — AND THERE WAS NOTHING TO LOOP ON. + // A retry needs a question it can ask, and `conn-status` used to have + // no answer: the engine wrote the HANDSHAKE's outcome into + // `conn_results` once and read it back forever, so a page holding a + // connection to a device that had been closed for an hour was told the + // same thing as one holding a live wire. Re-dialling on a timer against + // that would have been the double-dial #78's direction discipline + // exists to prevent — a second connection carrying the same + // subscriptions, silently. #113's engine half is what made this loop + // writable: every connection now carries a monitor that overwrites its + // entry with `Err("gone: …")` when the wire dies, and `connGone` above + // is the only gate that opens the re-dial path. THE DISCIPLINE IS + // UNCHANGED, it is merely now enforceable: a re-dial happens ONLY when + // the old connection has said it is dead. + // + // THE ROLE IS THE SAME ROLE, and it must be: reversing the direction is + // the healthy-looking silence #78 is about. So each tick re-enacts what + // this device did at ceremony time, reading its role out of the + // account's directory (each entry carries an ENDPOINT — where that + // device can be dialled — and an ENROLLED-BY — which device let it in): // // * MY ENROLLER, if I have one, is who I DIALLED then; I dial it // again. (READER DIALS.) @@ -4348,54 +4595,92 @@ async function startApp( // device may be shut, on a train, or opened tomorrow — so the retry // runs for as long as the page lives, and says so ONCE. A line per // attempt would be a page shouting a fact that has not changed. - - // AND WHAT THIS DOES NOT RECOVER, written down because the shape of - // the gap is not obvious and the workaround for it would be worse. // - // Only ONE side reloading is NOT recovered, when the side that - // reloaded is the one that ACCEPTS. Its own resume posts an acceptor - // and waits, correctly; but its peer — the reader — still holds a - // connection handle from before, and has no way to learn that the - // thing on the other end of it is gone. `conn-status` reports the - // outcome of the HANDSHAKE and is never invalidated afterwards - // (engine/guest/src/lib.rs:4407 writes it once — `s.conn_results.insert(id, - // outcome)`, with the error paths at :4343 and :4400 writing the same - // slot early — and :4413-4420's `conn_status` reads that slot back - // forever), and `sync-status` is one-shot per round rather than a - // subscription's health. So the reader has no evidence of staleness at - // all, and the only "fix" available to this file would be to re-dial - // on a timer — a second connection and a second set of subductions for - // the same pair, which is precisely the double-dialling the direction - // discipline exists to prevent. + // THE ENDPOINT IS THE OTHER CASUALTY, and this is the part that is easy + // to get wrong. MEASURED on this engine (throwaway probe, relay + // SIGKILL then restart on the same port): the relay's death does not + // merely kill the CONNECTIONS, it latches this device's own ENDPOINT + // `Closed` — a fresh `iroh-start` afterwards fails with + // `connect: Error::Closed` and keeps failing, no matter how healthy the + // relay is by then. A re-dial alone therefore heals nothing; the + // endpoint has to be rebound first, and after a rebind the dial lands + // in ~0.25s. + // + // THE REBIND IS DRIVEN BY EVIDENCE, NEVER BY SUSPICION. It would be + // easy to read "a connection went gone" as "rebind" — and wrong, badly: + // a rebind tears down the endpoint, which takes EVERY other connection + // on this page with it, including ones that are perfectly alive (the + // peer-vanish case, where the relay never went anywhere). So the sweep + // below only ever FORGETS a dead wire; the rebind stays where it + // already was, on the `Closed` error that the re-establish attempt + // itself raises (`isEndpointClosed` → `rebindEndpoint`, at most once + // per tick). The tick that notices the death is the tick that tries to + // repair it and learns the endpoint is gone too, which costs one extra + // cadence and buys the guarantee that a live wire is never collateral. // - // The both-sides case, which is the ordinary one (a user closes their - // laptop, then their phone), IS recovered: both readers come back with - // no handle to be misled by, and dial. + // TWO DEATHS, TWO CLOCKS, and the difference is worth carrying because + // it is the difference between "this feels instant" and "this takes + // most of a minute", with identical code on this side of the boundary: // - // The honest fix belongs in the engine — a `conn-status` that goes - // false when the connection drops — and until it exists this page - // cannot tell the difference between a healthy peer and a departed - // one. Filed as #113. + // * A RELAY DIES: the relay leg is a websocket over TCP, so the + // socket dies under this device and the endpoint learns + // SYNCHRONOUSLY. `gone:` in under a second, measured + // (demo/host/conn-gone-check.ts). It also latches this device's + // endpoint `Closed` — see the rebind note above. + // * A PEER VANISHES (its page reloads) while the relay stays up: + // nothing under THIS device dies, and a page that has been + // navigated away from sends no close frame, so the connection ends + // only when the QUIC IDLE TIMEOUT fires. Measured end to end: 35s + // from the peer's disappearance to this page having re-dialled and + // the data crossing, three runs, of which ~30s is that timeout + // (the e2e `one-sided-reload` scenario, which was `expected: "red"` + // until this loop existed and is now green). + // + // Both numbers belong to the pinned endpoint rather than to this page, + // which is why nothing here asserts either of them: the keeper waits + // for the marker however long it takes, and the scenarios that DO + // assert a bound say where the bound came from. + // + // AND THE LAST GAP, RECOVERED IN THE ENGINE, recorded because this + // page spent a round unable to close it from up here: a relay OUTAGE + // between two pages that both stay open. Everything in this file's + // vocabulary worked — both sides mark the wire gone, rebind, repost, + // re-dial, and report a live connection again within ~10s of the + // relay returning — and for one round the partition STILL did not + // converge, because subduction did not resume across the endpoint + // rebind: the engine's peer registry kept the dead transport and + // walked it first, forever (the four-link chain is written out in + // scenarios/relay-partition.ts's banner, wave 3). That was below this + // page's vocabulary and was fixed in the engine (the gone monitor now + // closes a dead connection's inbound queues, so subduction's own + // teardown runs); demo/host/rebind-sync-check.ts pinned it red→green, + // and `relay-partition`/`relay-partition-asym` are green end to end — + // heal measured at ~5s from the relay's return. /** Slow on purpose: the thing being waited for is another human * opening a browser. */ - const RESUME_TICK_MS = 5_000; + const KEEPER_TICK_MS = 5_000; /** Ten minutes, and it is a wait for a PERSON, not for a wire. */ const RESUME_POINTER_MS = 600_000; - let resumeWired = false; + let keeperRunning = false; - /** Wire a device that already holds the account. + /** Arm the wire-keeper. Idempotent, and that is what makes it safe to + * call from all three entry points: whichever one gets here first owns + * the loop, and the others become no-ops. * - * `needsTasks` is the un-dead-end: an account whose todo-list pointer - * has not reached this device yet. That used to park the page on "this - * account has no todo list yet" — a sentence that describes a - * PERMANENT state and was being said about a temporary one. The - * pointer is us-doc content, so it arrives exactly when the wire below - * comes up; the page waits for it instead of concluding from it. */ - const resumeWire = async (needsTasks: boolean) => { - if (resumeWired) return; - resumeWired = true; + * `needsTasks` is the un-dead-end, and it belongs to the resumed-boot + * caller alone: an account whose todo-list pointer has not reached this + * device yet. That used to park the page on "this account has no todo + * list yet" — a sentence that describes a PERMANENT state and was being + * said about a temporary one. The pointer is us-doc content, so it + * arrives exactly when the wire below comes up; the page waits for it + * instead of concluding from it. A ceremony caller passes false: it has + * just carried the pointer across itself. */ + const wireKeeper = async (needsTasks: boolean) => { + if (keeperRunning) return; + keeperRunning = true; + if (needsTasks) tasksHeld = false; const st = await conn.status(); // Hex both ways, lower-cased at the boundary: the worker's `meta` @@ -4405,7 +4690,8 @@ async function startApp( // deciding it has no role in its own account. const me = (st.agentId ?? "").toLowerCase(); if (!me) { - console.warn("[solo] resume wiring: this device has no recorded agent id"); + console.warn("[solo] wire-keeper: this device has no recorded agent id"); + keeperRunning = false; return; } @@ -4416,43 +4702,24 @@ async function startApp( status("waiting for your other device…"); }; - /** Dialled at most once — a second dial to the same peer is a second - * connection carrying the same subscriptions. */ - let dialled = false; - /** Which peers this device has already subscribed the todo list to. - * Keyed by agent id, hex. */ - const tasksWired = new Set(); - /** Whether the tasks partition is this device's to read. False only - * on the `needsTasks` path, until the pointer arrives and the - * adoption below runs. */ - let tasksHeld = !needsTasks; - - /** Subscribe the account's todo list to one peer. + /** Subscribe the account's todo list to one peer, if it is this + * device's to subscribe and this peer's wire does not already carry + * it. * * ONLY THE TASKS PARTITION, because only it needs asking: the engine * subscribes the us doc to every known peer itself on every pump * (usdoc.rs's `ensure_subscriptions`), and it has no name for this - * one. Same division of labour as adderWire's. */ + * one. Same division of labour as adderWire's — and the same + * `claimTasks` guard, which is what makes it safe for this loop and a + * still-retrying ceremony to be interested in the same peer. */ const wireTasks = async (peerHex: string) => { if (!tasksHeld || tasksWired.has(peerHex)) return; const list = await enqueue(() => driver.usPartitions()); const part = list.find((p) => p.name === TASKS_POINTER); if (!part) return; - tasksWired.add(peerHex); - try { - // #115: not wrapped in an outer `enqueue` — `subscribe` enqueues - // its own driver calls (see its definition), so wrapping it here - // too would nest one job inside another and self-deadlock the - // chain (see the `enqueue` footgun note above). - await subscribe(unhex(peerHex), part.id, "your todo list"); - usSynced = true; - console.log(`[solo] subduction re-wired: this device ⇄ ${peerHex.slice(0, 8)}…`); - } catch (e) { - // Back out of the set: a subscription that did not take must be - // retried, or this device is silently unsubscribed for ever. - tasksWired.delete(peerHex); - throw e; - } + await claimTasks(peerHex, unhex(peerHex), part.id); + usSynced = true; + console.log(`[solo] subduction re-wired: this device ⇄ ${peerHex.slice(0, 8)}…`); }; // THE POINTER ARM, when this device has no todo list yet. It runs @@ -4485,7 +4752,7 @@ async function startApp( })(); } - /** One attempt at everything, run again every `RESUME_TICK_MS`. Each + /** One attempt at everything, run again every `KEEPER_TICK_MS`. Each * step guards itself, so a tick that achieves half the wiring keeps * the half it got. */ const tick = async () => { @@ -4495,17 +4762,59 @@ async function startApp( // one to fail against. let rebound = false; const onError = async (e: unknown, what: string) => { - console.warn(`[solo] resume ${what}: ${err(e)}`); + console.warn(`[solo] wire-keeper ${what}: ${err(e)}`); if (rebound || !isEndpointClosed(e)) return; rebound = true; try { await rebindEndpoint(); } catch (e2) { - console.warn(`[solo] resume: rebinding the endpoint failed: ${err(e2)}`); + console.warn(`[solo] wire-keeper: rebinding the endpoint failed: ${err(e2)}`); } }; + // --- THE HEALTH SWEEP (#113), before anything is rebuilt --------- + // + // One cheap read per live handle, each its own millisecond-sized + // `enqueue` slot, and together they are the ONLY thing standing + // between this loop and the double-dial: everything below rebuilds + // what the sweep has just declared dead, and rebuilds nothing else. + // A wire that has not said `gone:` is left strictly alone, however + // long it has been quiet — silence is not evidence, and #78's + // silent double-dial is what happens when a page decides otherwise. + // + // A `failed` reading is passed to `onError` rather than acted on, + // because the one failure that matters here is this device's OWN + // endpoint having died (`Closed`), and that is a fact about the + // transport rather than about the peer: it is `rebindEndpoint`'s to + // answer, not the re-dial path's. + if (dialledPeer) { + const h = await probeConn(dialledPeer.conn); + if (h.kind === "gone") { + console.warn( + `[solo] the wire to ${dialledPeer.peerHex.slice(0, 8)}… is gone; re-dialling`, + ); + // Forgetting the SUBSCRIPTION with the connection is not + // bookkeeping tidiness: a subduction is a property of the conn + // it was started on, so it died with it, and re-subscribing on + // the fresh conn below is a first subscription rather than a + // second. + forgetDial(); + } else if (h.kind === "failed") await onError(h.error, "dial status"); + } + if (acceptorConn !== null) { + const h = await probeConn(acceptorConn); + if (h.kind === "gone") { + console.warn("[solo] the acceptor's wire is gone; reposting"); + // The children's dials will come back to the fresh acceptor — + // each child's own keeper is doing exactly what this one is — + // and this side re-subscribes them when they land. + forgetAcceptor(); + } else if (h.kind === "failed") await onError(h.error, "acceptor status"); + } - // WRITER ACCEPTS, first and always — see the section note. + // WRITER ACCEPTS, first and always — see the section note. When the + // sweep above just cleared the flag, this is where the repost + // happens, and where a `Closed` endpoint (the relay-death case) + // surfaces as the error that drives the rebind. try { await postAcceptor(); } catch (e) { @@ -4520,7 +4829,12 @@ async function startApp( // READER DIALS: my enroller is the device I dialled at ceremony // time, and an empty `enrolled-by` means I am the founding device // and never dialled anyone. - if (!dialled && mine && mine.enrolledBy !== "") { + // + // `dialledPeer === null` is the gate, and after #113 it means one + // of exactly three things: this page has never dialled, its dial + // failed, or the sweep above saw the old one die. It never means + // "it has been a while". + if (dialledPeer === null && mine && mine.enrolledBy !== "") { const enroller = devices.find( (d) => d.agentId.toLowerCase() === mine.enrolledBy.toLowerCase(), ); @@ -4531,14 +4845,17 @@ async function startApp( if (enroller && !enroller.revoked && enroller.endpoint !== "") { sayWaiting(); try { - await dialPeer(unhex(enroller.agentId), unhex(enroller.endpoint)); - dialled = true; + await dialOnce( + enroller.agentId.toLowerCase(), + unhex(enroller.agentId), + unhex(enroller.endpoint), + ); } catch (e) { await onError(e, "dial"); } } } - if (dialled && mine) { + if (dialledPeer && mine) { try { await wireTasks(mine.enrolledBy.toLowerCase()); } catch (e) { @@ -4556,11 +4873,15 @@ async function startApp( const children = devices.filter( (d) => !d.revoked && d.enrolledBy !== "" && d.enrolledBy.toLowerCase() === me, ); + // Remembered for the NEXT tick's sweep, which needs to know whose + // subscriptions to forget when the acceptor dies — by then the + // directory read has not happened yet. + childrenHex = children.map((d) => d.agentId.toLowerCase()); if (children.length > 0 && acceptorConn !== null) { sayWaiting(); let connected = false; try { - connected = Boolean(await driver.connStatus(acceptorConn)); + connected = Boolean(await enqueue(() => driver.connStatus(acceptorConn as number))); } catch (e) { await onError(e, "acceptor status"); } @@ -4579,9 +4900,12 @@ async function startApp( // NOT `until`, and not a bounded count. `poll` skips a tick whose // predecessor is still running, which is exactly right here: a dial // has its own 30s deadline inside it, and overlapping attempts would - // be several endpoints racing to reach one peer. + // be several endpoints racing to reach one peer. It is also what + // keeps the re-dial SINGLE-FLIGHT now that the loop runs for the life + // of the page: an attempt in progress means the next cadence is + // skipped, not queued behind it. void tick(); - poll(RESUME_TICK_MS, tick); + poll(KEEPER_TICK_MS, tick); }; const addTenant = visor.drawer.tenant<{ container: HTMLElement }>({ @@ -4781,15 +5105,15 @@ async function startApp( // would be the page confusing "not yet in sync" with "not yet // usable". await mountApp(); - void resumeWire(false); + void wireKeeper(false); } else { // NOT A DEAD END ANY MORE. An account with no todo list is still // not a first run — offering to create a SECOND account here would // be the page guessing at a state it does not understand — but nor // is it a state to park in. The pointer lives in the account's own - // document, so it arrives when the wire does; `resumeWire` says so + // document, so it arrives when the wire does; `wireKeeper` says so // on screen, waits, adopts, and mounts. - void resumeWire(true); + void wireKeeper(true); } } else { note("account:none"); @@ -4872,6 +5196,38 @@ async function startApp( * has an account at all. */ hasAccount: async () => (await us.usProfileGet()).ok, usSynced: () => usSynced, + /** WHAT THE WIRE-KEEPER BELIEVES, for a scenario that has to explain + * a failure rather than merely report one (#113). Read-only, and + * deliberately: nothing here lets a test dial, drop, or repost + * anything — it is the keeper's own account of which handles it holds + * and what `conn-status` says about each of them right now. + * + * `state` is the raw three-way the contract defines: "alive" (the + * handshake settled and the wire is last-known-good), "gone" (it came + * up and died — the marker the keeper re-dials on), "settling" (a + * handshake with no outcome yet), or the error text for anything + * else. A scenario reading "alive" on both sides while nothing + * crosses has learnt something quite different from one reading + * "gone" on a wire that is not being re-dialled. */ + wireHealth: async () => { + const look = async (conn: number | null) => { + if (conn === null) return null; + const h = await probeConn(conn); + return { + conn, + state: h.kind === "failed" ? h.message : h.kind, + peer: h.kind === "alive" ? h.peer : "", + }; + }; + return { + dialled: dialledPeer + ? { ...(await look(dialledPeer.conn)), peer: dialledPeer.peerHex.slice(0, 8) } + : null, + acceptor: await look(acceptorConn), + subscribed: [...tasksWired].map((p) => p.slice(0, 8)), + keeper: keeperRunning, + }; + }, /** THE DEVICE, as the store holds it. Nothing personal: an opaque * id, the tier, the policy and the rungs the picker reasons about. */ deviceId: () => conn.deviceId, diff --git a/demo/justfile b/demo/justfile index 0159ddef..f31a462b 100644 --- a/demo/justfile +++ b/demo/justfile @@ -50,6 +50,27 @@ pairing-bringup: translate deno check host/pairing-bringup.ts deno run -A host/pairing-bringup.ts +# #113's engine half: conn-status must report a wire that came up and +# later DIED, with the machine-readable `gone: ` marker the page half +# re-dials on. Needs NO `just infra` — the probe spawns its OWN relay on +# an ephemeral port precisely so it can kill it. Fast: the relay leg is a +# websocket, so killing the relay tears the socket down and both sides +# notice in ~0.1s. (The QUIC idle timeout governs only the shapes where +# no socket dies — a black-holing relay, a NAT drop — see the probe's +# banner, which measures rather than promises.) +conn-gone: translate + deno check host/conn-gone-check.ts + deno run -A host/conn-gone-check.ts + +# #113's RESIDUAL gap: noticing a dead wire is not the same as being +# able to USE its replacement. After a relay outage both sides rebind +# their endpoint and re-dial, conn-status goes live in seconds — and +# then sync-start never settles on the dialling side. Own relay child, +# ephemeral port, same rig as conn-gone. +rebind-sync: translate + deno check host/rebind-sync-check.ts + deno run -A host/rebind-sync-check.ts + # The #20 G5 kill-and-resume beat: the engine checkpoints into a real # directory (@polyengine/wasi's `filesystem-node` backend) and a FRESH # DENO PROCESS resumes from it. Needs no infra at all — the assertion is diff --git a/engine/guest/src/lib.rs b/engine/guest/src/lib.rs index 813e015a..dab3ef1e 100644 --- a/engine/guest/src/lib.rs +++ b/engine/guest/src/lib.rs @@ -609,6 +609,14 @@ struct State { nonce_cache: Rc, proto: Rc, conn_results: HashMap>, + /// The INBOUND channel senders of every queue this connection feeds + /// — the subduction transport's, and the keyhive wire's once the + /// handshake mints it. Held for exactly one purpose: closing them + /// when the wire dies, so `recv` on the other end FAILS instead of + /// parking forever. See `conn_gone_monitor` for why a transport that + /// cannot fail is worse than no transport at all. + conn_inbound: HashMap>>>, + syncs: HashMap>, endpoint: Option>, /// The relay this endpoint bound to. Pairing has no relay hint in the @@ -833,6 +841,164 @@ async fn iroh_reader(in_tx: async_channel::Sender>, recv: RecvStream, se } } +/// The wire's obituary (#113). `conn-status` used to be write-once: the +/// handshake outcome went into `conn_results` and was read back forever, +/// so a connection that came up and later DIED was indistinguishable +/// from a healthy one — and a page with no liveness signal may never +/// safely re-dial, because a second dial to the same peer is a second +/// connection carrying the same subscriptions (#78's direction +/// discipline). Nothing else observes the death: `iroh_writer` breaks on +/// a write error and `iroh_reader` breaks on EOF, both silently, into +/// their own tasks. So one more task per connection does nothing but +/// wait for the end and write it down. +/// +/// `wait-closed` is the notification, and it was always there — it needed +/// plumbing, not a WIT change (guest/wit/deps/polymorph-iroh/iroh.wit:425 +/// — async, resolving with the peer's `close-info` when an application +/// close arrived and `none` when the connection ended any other way, and +/// LATCHED: once closed it resolves immediately with the same value any +/// number of times). It is a method on `connection`, not on either +/// stream, so holding a second `Rc` here does not contend +/// with the reader/writer tasks' borrows of `SendStream`/`RecvStream` — +/// those are separate resources handed out by `open-bi`/`accept-bi`. +/// +/// TWO RULES, both about not lying: +/// +/// 1. It only overwrites an `Ok(peer)`. An existing `Err` is a +/// wire-setup or handshake failure, which says strictly more than +/// "gone", and clobbering it would turn a diagnosis into a shrug. +/// +/// A MISSING entry means the wiring task has not settled yet, and it +/// is NOT a reason to give up: real awaits sit between the handshake +/// returning `Ok` and the final insert (`proto.add_peer`, +/// `refreshed_sync` — seconds wide), so a connection that dies inside +/// that window would find no entry, write nothing, exit — and then +/// the wiring task would insert its now-stale `Ok`, latching a dead +/// wire ALIVE FOREVER, which is the exact bug this monitor exists to +/// kill. So it WAITS instead: `wait-closed` is latched, so the news +/// keeps until there is somewhere to put it, and the loop below +/// breathes until an entry appears (or the connection is forgotten +/// entirely, which is the other way out). Belt-and-braces, the final +/// insert also consults `conn.state()` — see the wiring task — so +/// the window is closed from both ends. +/// 2. The marker is the literal prefix `gone: `, and engine.wit's +/// `conn-status` doc names it as machine-readable. The page half +/// matches on it; it is not free text to be reworded. +/// +/// THE OTHER HALF, and the more consequential one: this monitor also +/// CLOSES THE WIRE'S INBOUND QUEUES, because until #113's follow-up the +/// engine was lying to subduction about liveness and that lie was worth +/// more than the missing status. +/// +/// `QueueTransport` (above) holds its OWN `in_tx` alongside `in_rx`, so +/// `recv_bytes` could never fail: when `iroh_reader` hit EOF it dropped +/// only its clone of the sender, the channel stayed open, and the +/// transport went on politely waiting for frames from a socket that no +/// longer existed. `send_bytes` was no better — `out_rx` is held by the +/// transport too, so writes into a dead wire SUCCEED into a void. +/// +/// Subduction's teardown is driven entirely by a connection's reader +/// failing (subduction_core/src/connection/manager.rs:289-332 spawns a +/// `connection_loop` per connection whose exit is the only thing that +/// posts to `connection_closed`, consumed at subduction.rs:3244 where +/// `remove_connection` finally runs). A transport that cannot fail +/// therefore never gets removed — and `add_connection` APPENDS rather +/// than replaces (`register_connection`, subduction.rs:758-791: the new +/// connection is pushed onto the peer's list and a second multiplexer +/// alongside it), so after an endpoint rebind the peer owns a dead +/// connection at index 0 and the live one at index 1. `sync_with_peer` +/// walks that list IN ORDER (subduction.rs:2190) and calls on the dead +/// one first; with this engine's `NeverTimeout` (see it above — the +/// `Timeout` impl that never fires) that call parks forever and the +/// live connection is never reached. Net effect before this fix: after +/// a relay outage, `conn-status` said LIVE within a second and no sync +/// ever completed again, for the life of the page. +/// +/// The asymmetry that made it confusing: only the side that CALLS +/// `sync_with_peer` walks the stale list. An acceptor answering an +/// incoming request replies on the connection the message arrived on, +/// so it settles normally — which is why `one-sided-reload` was green +/// throughout and only the dialling side ever hung. +/// +/// Closing the inbound senders is the whole repair: `recv_bytes` starts +/// failing, `connection_loop` exits, subduction removes the connection +/// and its multiplexer by its own machinery, and the next +/// `sync_with_peer` finds only the live one. No upstream change, and +/// nothing here reaches into subduction's registries behind its back. +/// +/// WHAT ACTUALLY FIRES THIS. The obvious theory is the QUIC IDLE +/// TIMEOUT — an idle connection sees no error, because nobody writes, +/// so nobody learns the path is gone — and for a path that silently +/// stops carrying packets (a black-holing relay, a NAT drop) that is +/// indeed what ends it: tens of seconds, a property of the pinned +/// endpoint (jsr:@polymorph/iroh@0.3.0), not a promise this engine +/// makes or can tighten. But the case the gate exercises is FASTER for +/// a structural reason: a relay-dialed connection's relay leg is a +/// websocket over TCP, so killing the relay process closes that socket +/// and the endpoint learns synchronously — `demo/host/conn-gone-check.ts` +/// measures under a second, both sides, with `none` for the close-info +/// (a peer that vanished sent no close frame). Callers should still +/// treat "gone" as eventually-consistent on the order of tens of +/// seconds: they cannot tell which mechanism they are about to get. +async fn conn_gone_monitor(id: u32, conn: Rc) { + let info = conn.wait_closed().await; + let why = gone_reason(info); + + // BEAT ONE, and the one that actually heals anything: TELL THE + // QUEUES THE TRUTH. See the "THE OTHER HALF" note above — until + // these channels close, `QueueTransport::recv_bytes` parks forever + // on a wire nobody will ever write to again, subduction never + // learns the connection died, and its replacement is unusable. + // + // Closing a `Sender` closes the channel for the receiver too, but + // only AFTER the queue drains — frames that arrived before the wire + // died are still delivered, so this loses no data. Done before the + // status write, and unconditionally, because it is repair rather + // than reporting: it must happen even when `conn_results` already + // holds an `Err` that rule 1 will decline to overwrite. + let _ = with_state(|s| { + for tx in s.conn_inbound.remove(&id).unwrap_or_default() { + tx.close(); + } + }); + + // BEAT TWO: rule 1's wait. The cap is a runaway guard, not a + // deadline: every path through the wiring task inserts SOMETHING, so + // in practice the first or second pass lands. If it somehow never + // does, giving up quietly is better than a task spinning for the + // life of the page. + for _ in 0..10_000 { + let settled = with_state(|s| match s.conn_results.get(&id) { + Some(Ok(_)) => { + s.conn_results.insert(id, Err(why.clone())); + true + } + // An Err is already more informative than "gone" (rule 1). + Some(Err(_)) => true, + // Nothing recorded yet: keep the news until the wiring task + // settles, unless the connection has been forgotten outright. + None => !s.iroh_conns.contains_key(&id), + }); + match settled { + Ok(true) | Err(_) => return, + Ok(false) => breathe().await, + } + } +} + +/// The `gone: ` message for a `wait-closed` outcome. Shared with the +/// wiring task's `state()` belt-and-braces check, which has no +/// `close-info` to hand and must still say the same kind of thing. +fn gone_reason(info: Option) -> String { + match info { + Some(ci) if !ci.reason.is_empty() => { + format!("gone: peer closed, code {}: {}", ci.code, ci.reason) + } + Some(ci) => format!("gone: peer closed, code {}", ci.code), + None => "gone: transport closed (no close frame)".to_string(), + } +} + // --- the bucket path (#19's pull layer; adapted from spikes/storage) --- #[derive(Serialize, Deserialize)] @@ -3652,6 +3818,7 @@ fn finish_init( nonce_cache: Rc::new(NonceCache::default()), proto, conn_results: HashMap::new(), + conn_inbound: HashMap::new(), syncs: HashMap::new(), endpoint: None, relay_url: None, @@ -4638,7 +4805,19 @@ impl DriverGuest for Component { let transport = QueueTransport::new(id); wit_bindgen::spawn_local(iroh_writer(transport.out_rx.clone(), s_send)); wit_bindgen::spawn_local(iroh_reader(transport.in_tx.clone(), s_recv, s_seed)); - let _ = with_state(|s| s.iroh_conns.insert(id, Rc::new(conn))); + let conn = Rc::new(conn); + // Registered BEFORE the monitor is spawned: the monitor's + // first act is to close whatever is registered here, and a + // connection that is already dead by the time we get this far + // must not find an empty list and leave the queue open. + let _ = with_state(|s| { + s.iroh_conns.insert(id, conn.clone()); + s.conn_inbound + .entry(id) + .or_default() + .push(transport.in_tx.clone()); + }); + wit_bindgen::spawn_local(conn_gone_monitor(id, conn.clone())); let outcome = subduction_handshake( transport, @@ -4660,7 +4839,27 @@ impl DriverGuest for Component { let (kh_out_tx, kh_out_rx) = async_channel::unbounded(); let (kh_in_tx, kh_in_rx) = async_channel::unbounded(); wit_bindgen::spawn_local(iroh_writer(kh_out_rx, k_send)); - wit_bindgen::spawn_local(iroh_reader(kh_in_tx, k_recv, k_seed)); + wit_bindgen::spawn_local(iroh_reader(kh_in_tx.clone(), k_recv, k_seed)); + // The keyhive stream rides the SAME connection, so + // it dies with it — and its receive loop below + // (`while let Ok(msg) = recv_wire.recv()`) parks on + // the same never-closing channel the subduction + // transport did. Same registration, same repair. + // + // A monitor that already fired (the connection died + // during the handshake) has drained and removed the + // list, so pushing here would leak an open queue: + // close immediately in that case instead. + let kh_registered = with_state(|s| match s.conn_inbound.get_mut(&id) { + Some(v) => { + v.push(kh_in_tx.clone()); + true + } + None => false, + }); + if !matches!(kh_registered, Ok(true)) { + kh_in_tx.close(); + } let kh_peer = KeyhivePeerId::from_bytes(peer32); let kh_wire = KhWire { peer: kh_peer.clone(), @@ -4694,6 +4893,19 @@ impl DriverGuest for Component { } } } + // BELT-AND-BRACES on the monitor's rule 1 (see + // `conn_gone_monitor`): everything above this line — the + // handshake, `proto.add_peer`, `refreshed_sync` — is seconds + // wide, and a connection that died inside that window would + // be latched ALIVE by this very insert. `state()` is latched + // `closed` (iroh.wit:334), so asking it costs nothing and + // catches the case without waiting for anything. + let outcome = match outcome { + Ok(_) if conn.state() == polymorph::iroh::endpoint::ConnectionState::Closed => { + Err(gone_reason(None)) + } + other => other, + }; let _ = with_state(|s| s.conn_results.insert(id, outcome)); }); diff --git a/engine/guest/wit/engine.wit b/engine/guest/wit/engine.wit index 23a20cfc..b93daa79 100644 --- a/engine/guest/wit/engine.wit +++ b/engine/guest/wit/engine.wit @@ -496,6 +496,28 @@ interface driver { iroh-bind: async func(relay-url: string) -> result; iroh-start: async func(initiator: bool, peer-endpoint-id: list, relay-url: string, expected-peer: list) -> result; + /// The connection's liveness, in three distinguishable answers + /// (#113 — this used to be write-once, reporting the handshake + /// outcome forever): + /// + /// - `ok(some(peer))`: the handshake succeeded and the wire was + /// last known ALIVE. `peer` is the authenticated peer id, hex. + /// - `ok(none)`: no such connection id. + /// - `err(e)` where `e` starts with the literal prefix `"gone: "`: + /// the wire came UP and later DIED. Latched — it never returns + /// to `ok`, because this id names that one dead connection; a + /// re-dial mints a NEW conn id. The rest of the message is + /// whatever the close carried (the peer's application close + /// code and reason when one arrived, else an honest note that + /// the transport just ended). + /// - any other `err(e)`: the HANDSHAKE ITSELF failed (wire setup, + /// peer mismatch, protocol error). Nothing ever came up. + /// + /// THE `"gone: "` PREFIX IS THE MACHINE-READABLE MARKER. Callers + /// that re-dial match on it to tell "this wire died, dial again" + /// from "this dial never worked, and dialling again the same way + /// will not work either". It is a contract, not phrasing: do not + /// rework it into free-form prose. conn-status: async func(conn: u32) -> result, string>; sync-start: async func(peer: list, tree: list, subscribe: bool) -> result; sync-status: async func(handle: u32) -> result, string>;