fix(persistence): fail fast when the shared migration ledger diverges

apps/server/src/persistence/ForkMigrations.test.ts

@@ -9,6 +9,8 @@ import {
   forkMigrationEntries,
   realignSharedMigrationLedger,
   runForkMigrations,
+  SharedMigrationLedgerMismatchError,
+  verifySharedMigrationLedger,
 } from "./ForkMigrations.ts";
 import * as NodeSqliteClient from "./NodeSqliteClient.ts";
 
@@ -54,6 +56,7 @@ layer("fresh install", (it) => {
       const sql = yield* SqlClient.SqlClient;
 
       yield* realignSharedMigrationLedger();
+      yield* verifySharedMigrationLedger();
       yield* runMigrations();
       yield* runForkMigrations();
 
@@ -72,6 +75,100 @@ layer("fresh install", (it) => {
   );
 });
 
+// Each layer() block gets its own in-memory database, so every ledger scenario
+// needs its own block rather than sharing one with the others.
+layer("verifySharedMigrationLedger on a fresh database", (it) => {
+  it.effect("passes when the ledger table does not exist yet", () =>
+    Effect.gen(function* () {
+      yield* verifySharedMigrationLedger();
+    }),
+  );
+});
+
+layer("verifySharedMigrationLedger after the shared chain runs", (it) => {
+  it.effect("passes, and keeps passing with ids beyond this build's chain", () =>
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient;
+
+      yield* runMigrations();
+      yield* runForkMigrations();
+      yield* verifySharedMigrationLedger();
+
+      yield* sql`
+        INSERT INTO effect_sql_migrations (migration_id, name, created_at)
+        VALUES (${UPSTREAM_MAX + 1}, 'SomeFutureMigration', CURRENT_TIMESTAMP)
+      `;
+
+      yield* verifySharedMigrationLedger();
+    }),
+  );
+});
+
+layer("verifySharedMigrationLedger on a partially migrated database", (it) => {
+  it.effect("passes when the ledger stops partway through the chain", () =>
+    Effect.gen(function* () {
+      yield* runMigrations({ toMigrationInclusive: 20 });
+      yield* verifySharedMigrationLedger();
+    }),
+  );
+});
+
+layer("verifySharedMigrationLedger with a divergent branch's chain", (it) => {
+  it.effect("fails, naming the migrations that would be skipped", () =>
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient;
+
+      // Reproduces ~/.t3/dev/state.sqlite: the shared chain ran through 32, then
+      // a branch that numbered its OrchestrationV2 chain at 33+ took over, so
+      // this build's 33.. never ran and their columns are missing.
+      yield* runMigrations({ toMigrationInclusive: 32 });
+      const divergent = ["OrchestrationV2", "OrchestrationV2Subagents", "ScheduledTasks"];
+      for (const [index, name] of divergent.entries()) {
+        yield* sql`
+          INSERT INTO effect_sql_migrations (migration_id, name, created_at)
+          VALUES (${33 + index}, ${name}, CURRENT_TIMESTAMP)
+        `;
+      }
+
+      const error = yield* Effect.flip(verifySharedMigrationLedger());
+
+      assert.instanceOf(error, SharedMigrationLedgerMismatchError);
+      assert.strictEqual(error.latestLedgerId, 35);
+      assert.deepStrictEqual(
+        error.skipped.map(({ id }) => id),
+        [33, 34, 35],
+      );
+      assert.deepStrictEqual(error.skipped[0], {
+        id: 33,
+        expected: "ProjectionThreadsSettled",
+        recorded: "OrchestrationV2",
+      });
+      assert.include(error.message, "ProjectionThreadsSettled");
+      assert.include(error.message, "OrchestrationV2");
+      assert.include(error.message, "T3CODE_HOME");
+    }),
+  );
+});
+
+layer("verifySharedMigrationLedger with a gap in the ledger", (it) => {
+  it.effect("reports the missing id as a skipped migration", () =>
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient;
+
+      yield* runMigrations({ toMigrationInclusive: 20 });
+      yield* sql`DELETE FROM effect_sql_migrations WHERE migration_id = 15`;
+
+      const error = yield* Effect.flip(verifySharedMigrationLedger());
+
+      assert.instanceOf(error, SharedMigrationLedgerMismatchError);
+      assert.deepStrictEqual(error.skipped, [
+        { id: 15, expected: "ProjectionTurnsSourceProposedPlan", recorded: null },
+      ]);
+      assert.include(error.message, "no row");
+    }),
+  );
+});
+
 layer("legacy full install", (it) => {
   it.effect("realigns a ledger that ran the old fork chain through upstream 35 (legacy 36)", () =>
     Effect.gen(function* () {
@@ -97,6 +194,7 @@ layer("legacy full install", (it) => {
       `;
 
       yield* realignSharedMigrationLedger();
+      yield* verifySharedMigrationLedger();
       yield* runMigrations();
       yield* runForkMigrations();
 
@@ -156,6 +254,7 @@ layer("legacy mid-history install", (it) => {
       `;
 
       yield* realignSharedMigrationLedger();
+      yield* verifySharedMigrationLedger();
       yield* runMigrations();
       yield* runForkMigrations();
 

apps/server/src/persistence/ForkMigrations.ts

@@ -20,8 +20,10 @@
 import * as Migrator from "effect/unstable/sql/Migrator";
 import * as Layer from "effect/Layer";
 import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
 import * as SqlClient from "effect/unstable/sql/SqlClient";
 
+import { migrationManifest } from "./Migrations.ts";
 import Migration0001 from "./Migrations/fork/001_ProjectionThreadParent.ts";
 
 export const FORK_MIGRATIONS_TABLE = "trogonstack_fork_migrations";
@@ -62,6 +64,93 @@ export const runForkMigrations = Effect.fn("runForkMigrations")(function* ({
   return executedMigrations;
 });
 
+export class SharedMigrationLedgerMismatchError extends Schema.TaggedErrorClass<SharedMigrationLedgerMismatchError>()(
+  "SharedMigrationLedgerMismatchError",
+  {
+    latestLedgerId: Schema.Number,
+    skipped: Schema.Array(
+      Schema.Struct({
+        id: Schema.Number,
+        expected: Schema.String,
+        recorded: Schema.NullOr(Schema.String),
+      }),
+    ),
+    unrecognized: Schema.Array(Schema.Struct({ id: Schema.Number, recorded: Schema.String })),
+  },
+) {
+  override get message(): string {
+    const skipped = this.skipped
+      .map(
+        ({ id, expected, recorded }) =>
+          `  ${id}: expected '${expected}', ledger has ${recorded === null ? "no row" : `'${recorded}'`}`,
+      )
+      .join("\n");
+    const unrecognized =
+      this.unrecognized.length === 0
+        ? ""
+        : `\nLedger ids this build does not know:\n${this.unrecognized
+            .map(({ id, recorded }) => `  ${id}: '${recorded}'`)
+            .join("\n")}`;
+
+    return [
+      `Shared migration ledger disagrees with this build's migration chain, so ${this.skipped.length} migration(s) would be skipped and their schema changes would be missing.`,
+      `The ledger's highest id is ${this.latestLedgerId}, and the migrator only runs ids above that.`,
+      `\nMigrations that would be skipped:\n${skipped}${unrecognized}`,
+      `\nThis database was migrated by a branch whose migration chain diverged from this one. Give this checkout its own T3 home (T3CODE_HOME or --home-dir), or reset/reseed this database. Do not renumber the ledger by hand.`,
+    ].join("\n");
+  }
+}
+
+/**
+ * Fail startup when the shared ledger cannot describe this build's chain.
+ *
+ * `Migrator` decides what to run purely by id (`currentId <= latestMigrationId`
+ * is skipped) and never compares names, so a ledger written by a branch with
+ * different numbering silently skips migrations and surfaces later as an
+ * unrelated "no such column" query failure. Sharing one T3 home across branches
+ * is normal here, so this turns that into an actionable startup error.
+ *
+ * Runs after `realignSharedMigrationLedger`, which repairs the one legacy
+ * divergence the fork itself shipped.
+ */
+export const verifySharedMigrationLedger = Effect.fn("verifySharedMigrationLedger")(function* () {
+  const sql = yield* SqlClient.SqlClient;
+
+  const tables = yield* sql<{ readonly name: string }>`
+      SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'effect_sql_migrations'
+    `;
+  if (tables.length === 0) {
+    return;
+  }
+
+  const rows = yield* sql<{ readonly migration_id: number; readonly name: string }>`
+      SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id
+    `;
+  if (rows.length === 0) {
+    return;
+  }
+
+  const recordedById = new Map(rows.map((row) => [row.migration_id, row.name]));
+  const latestLedgerId = Math.max(...rows.map((row) => row.migration_id));
+
+  const skipped = migrationManifest
+    .filter(([id, name]) => id <= latestLedgerId && recordedById.get(id) !== name)
+    .map(([id, name]) => ({ id, expected: name, recorded: recordedById.get(id) ?? null }));
+
+  if (skipped.length === 0) {
+    return;
+  }
+
+  const expectedIds = new Set<number>(migrationManifest.map(([id]) => id));
+  const unrecognized = rows
+    .filter((row) => !expectedIds.has(row.migration_id))
+    .map((row) => ({ id: row.migration_id, recorded: row.name }));
+
+  return yield* Effect.fail(
+    new SharedMigrationLedgerMismatchError({ latestLedgerId, skipped, unrecognized }),
+  );
+});
+
 /**
  * Realign the shared `effect_sql_migrations` ledger for installs that ran
  * the fork's old migration chain, where ProjectionThreadParent shipped as

apps/server/src/persistence/Layers/Sqlite.ts

@@ -6,7 +6,11 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";
 import type { SqlError } from "effect/unstable/sql/SqlError";
 
 import { runMigrations } from "../Migrations.ts";
-import { realignSharedMigrationLedger, runForkMigrations } from "../ForkMigrations.ts";
+import {
+  realignSharedMigrationLedger,
+  runForkMigrations,
+  verifySharedMigrationLedger,
+} from "../ForkMigrations.ts";
 import { ServerConfig } from "../../config.ts";
 
 type RuntimeSqliteLayerConfig = {
@@ -39,6 +43,7 @@ const setup = Layer.effectDiscard(
     yield* sql`PRAGMA foreign_keys = ON;`;
     yield* sql`PRAGMA journal_mode = WAL;`;
     yield* realignSharedMigrationLedger();
+    yield* verifySharedMigrationLedger();
     yield* runMigrations();
     yield* runForkMigrations();
   }),

docs/fork/0006-fork-migration-ledger.md

@@ -0,0 +1,41 @@
+# 0006: Fork schema on its own migration ledger
+
+- PR: [TrogonStack/t3code#13](https://github.com/TrogonStack/t3code/pull/13),
+  [TrogonStack/t3code#16](https://github.com/TrogonStack/t3code/pull/16)
+- Status: active
+
+## What you can do now
+
+- Fork-only schema lands on a migration chain of its own, so upstream's
+  migration numbering is never taken by the fork and a sync no longer has to
+  make room for fork ids.
+- An install that ran the fork's older history is brought back onto upstream
+  numbering on the next start, with no manual database surgery.
+- Starting a server against a database that some other branch migrated stops
+  right away and names every migration that would have been skipped, instead
+  of booting on a schema that only looks current and then failing in an
+  unrelated query.
+
+## Why
+
+The fork carries schema upstream does not have. Numbering it inside the
+shared chain permanently offsets every upstream id after it, which turns each
+sync into a conflict over files that should never diverge.
+
+The startup check exists because one T3 home routinely gets shared across
+branches here. Migrations are chosen by id alone, so a home that another
+branch already migrated makes this build skip its own migrations while the
+ledger still reads as fully migrated. That failure then surfaces far from its
+cause, as a query error and a restart loop, which is an expensive way to
+learn that the home directory belongs to somewhere else.
+
+## Upstream considerations
+
+The second chain only exists because the fork carries its own schema, so it
+is not something upstream wants as it stands, and it is deliberately built so
+the shared migration files stay byte-identical to upstream forever. New
+fork-only schema belongs on the fork chain, never on the shared one.
+
+The startup check is not fork-specific and upstream may want it on its own
+merits, since anyone running several checkouts against one home can hit the
+same silent skip.

docs/fork/README.md

@@ -27,7 +27,8 @@ Each entry uses these sections:
 
 ## Ledger
 
-| #    | Divergence                                                                                     | PR                                                   | Status |
-| ---- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------- | ------ |
-| 0003 | [Native subagent threads for Claude orchestrators](./0003-native-subagent-threads.md)          | [#3](https://github.com/TrogonStack/t3code/pull/3)   | active |
-| 0007 | [API-key Codex installs are not reported as broken](./0007-codex-api-key-auth-is-supported.md) | [#15](https://github.com/TrogonStack/t3code/pull/15) | active |
+| #    | Divergence                                                                                     | PR                                                                                                         | Status |
+| ---- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------ |
+| 0003 | [Native subagent threads for Claude orchestrators](./0003-native-subagent-threads.md)          | [#3](https://github.com/TrogonStack/t3code/pull/3)                                                         | active |
+| 0006 | [Fork schema on its own migration ledger](./0006-fork-migration-ledger.md)                     | [#13](https://github.com/TrogonStack/t3code/pull/13), [#16](https://github.com/TrogonStack/t3code/pull/16) | active |
+| 0007 | [API-key Codex installs are not reported as broken](./0007-codex-api-key-auth-is-supported.md) | [#15](https://github.com/TrogonStack/t3code/pull/15)                                                       | active |