References on frames and threading headers in the list projection

ARCHITECTURE.md

@@ -106,7 +106,8 @@ never in a database of their own. Every row in either belongs to exactly one
 ```
 principal_mail   the message as delivered. IMMUTABLE.
                  id, tenant_id, principal_id, address, direction, raw,
-                 subject, from_address, message_key, refs, created_at
+                 subject, from_address, message_id, in_reply_to,
+                 message_key, refs, created_at
 
 mailbox          the management layer, keyed by mail id. Mutable.
                  read_at, archived_at, trashed_at           (universal)

CHANGELOG.md

@@ -11,6 +11,37 @@ always called out under their own heading.
 
 ### Added
 
+- **Threading headers on the frame and in the list projection.**
+  `buildMailFrame` accepts `references` — the thread's ancestry, oldest
+  first — and emits it as a folded `References:` header; `In-Reply-To`
+  stays the single immediate parent. Every threading value must be a
+  bracketed msg-id (`<local@domain>`); anything else is a `RangeError` at
+  the builder, because a frame is frozen at rest and an unthreadable
+  header written today is unthreadable forever. `decodeMailFrame` now
+  returns `messageId`, `inReplyTo` (both `string | null`) and
+  `references` (`string[]`, oldest first) parsed alongside the header map.
+  `principal_mail` gains `message_id` and `in_reply_to` as cached
+  columns, populated on the `writeMailboxMessage`, `deliverInboxItems`
+  and `createMailboxPersist` paths, and `MailboxMessage` gains an
+  optional `inReplyTo` — so a client threads an inbox page from the list
+  projection alone, which never loads `raw`. `WriteMailboxMessageArgs`
+  and `InboxItem` accept `inReplyTo` and `references`. Migration
+  `0002_mail_threading_headers` adds both columns and backfills them from
+  each existing row's `raw`, so threading does not silently begin at the
+  upgrade. The backfill slices the header section out of `raw` at the
+  `bytea` level and strips any NUL byte from that slice before decoding
+  it, so a legacy frame with a NUL anywhere in its bytes (a binary
+  attachment, most commonly) cannot abort the migration. `assertMsgId`
+  accepts a quoted-string local part (`<"john doe"@example.com>`,
+  RFC 5322 `obs-id-left`) in addition to a dot-atom one. The cached
+  `in_reply_to` is normalized once on the way in — trimmed and
+  newline-flattened the same way `buildMailFrame` normalizes the header —
+  so the list and detail projections of the same message agree; on the
+  transport dual-write path (`createMailboxPersist`), `in_reply_to` caches
+  the first bracketed msg-id found in a decoded `In-Reply-To:` header, or
+  `null`, matching what the 0002 backfill derives from the same header
+  text rather than caching an unbracketed or multi-id header verbatim.
+
 - **Live events name the operation that fired.** `MailboxEvent` gains an
   optional `op` (`MailboxEventOp`: `create`, `mark_read`, `mark_unread`,
   `trash`, `archive`, `restore`, `enrich`, `assign`) alongside the existing

src/frame.test.ts

@@ -66,6 +66,82 @@ describe("buildMailFrame headers", () => {
     expect(h.has("bcc")).toBe(false);
   });
 
+  test("References is emitted oldest first, folded, and round-trips", () => {
+    // A three-deep chain is the shortest one where order and folding both
+    // matter: a client walks References oldest-first to place a reply under its
+    // root, and RFC 2822 caps a header line at 78 characters — which a chain
+    // passes within a few ids.
+    const chain = [
+      "<root-message-of-the-thread@example.com>",
+      "<second-message-of-the-thread@example.com>",
+      "<third-message-of-the-thread@example.com>",
+    ];
+    const raw = frame({ references: chain, inReplyTo: chain[2] });
+    const text = new TextDecoder().decode(raw);
+    // Folded: continuation lines begin with the single space RFC 2822 requires.
+    expect(text).toContain("\r\n <");
+    for (const line of text.split("\r\n")) {
+      expect(line.length).toBeLessThanOrEqual(78);
+    }
+    const decoded = decodeMailFrame(raw);
+    expect(decoded?.references).toEqual(chain);
+    // In-Reply-To stays a SINGLE msg-id — the immediate parent, not the chain.
+    expect(decoded?.inReplyTo).toBe(chain[2]!);
+    expect(decoded?.messageId).toBe("<fixed-id@example.com>");
+  });
+
+  test("no References header when the chain is absent or empty", () => {
+    expect(headers(frame()).has("references")).toBe(false);
+    expect(headers(frame({ references: [] })).has("references")).toBe(false);
+    expect(decodeMailFrame(frame())?.references).toEqual([]);
+    expect(decodeMailFrame(frame())?.inReplyTo).toBeNull();
+  });
+
+  test("a frame with no Message-ID decodes to a null messageId", () => {
+    const raw = new TextEncoder().encode(
+      "From: a@b.c\r\nTo: d@e.f\r\nSubject: s\r\n\r\nbody\r\n",
+    );
+    const decoded = decodeMailFrame(raw);
+    expect(decoded?.messageId).toBeNull();
+    expect(decoded?.inReplyTo).toBeNull();
+    expect(decoded?.references).toEqual([]);
+  });
+
+  test("a threading value that is not a bracketed msg-id is refused", () => {
+    // A frame is frozen at rest: an unthreadable header written today is
+    // unthreadable forever, so this is refused at the builder rather than
+    // stored and discovered by an MTA.
+    expect(() => frame({ messageId: "fixed-id@example.com" })).toThrow(
+      RangeError,
+    );
+    expect(() => frame({ messageId: "<<double@example.com>>" })).toThrow(
+      RangeError,
+    );
+    expect(() => frame({ inReplyTo: "parent@example.com" })).toThrow(RangeError);
+    expect(() =>
+      frame({ references: ["<ok@example.com>", "not-an-id"] }),
+    ).toThrow(RangeError);
+    expect(() => frame({ references: ["<no-domain>"] })).toThrow(RangeError);
+  });
+
+  test("a quoted local part (RFC 5322 obs-id-left) is accepted", () => {
+    // `"john doe"@example.com` is a legal (if obsolete-syntax) local part;
+    // widened rather than documented as a limitation because a quoted local
+    // part is real, if rare, on host-supplied threading headers.
+    const h = headers(
+      frame({ messageId: '<"john doe"@example.com>' }),
+    );
+    expect(h.get("message-id")).toBe('<"john doe"@example.com>');
+    expect(
+      () => frame({ inReplyTo: '<"a b"@example.com>' }),
+    ).not.toThrow();
+    // A quoted part still cannot contain a bare `<` or `>`, and an unescaped
+    // trailing quote must close the string before the `@`.
+    expect(() => frame({ messageId: '<"open@example.com>' })).toThrow(
+      RangeError,
+    );
+  });
+
   test("the body round-trips through decodeMailFrame", () => {
     const decoded = decodeMailFrame(frame({ body: "line one\nline two" }));
     expect(decoded?.body).toBe("line one\nline two");

src/frame.ts

@@ -35,6 +35,70 @@ export function generateMailboxMessageId(fromAddress: string): string {
   return `<${crypto.randomUUID()}@${domain === "" ? MESSAGE_ID_FALLBACK_DOMAIN : domain}>`;
 }
 
+/**
+ * A msg-id as every threading header carries it: exactly one pair of angle
+ * brackets, an `@`, and no whitespace outside a quoted local part. The same
+ * shape `@intx/mime`'s `generateMessageId` and `generateMailboxMessageId`
+ * return.
+ *
+ * The local part (`id-left`) accepts either a dot-atom token (no `<`, `>`,
+ * whitespace, or `"`) or an RFC 5322 quoted-string (`"..."`, backslash-escaped
+ * quotes and backslashes allowed inside) — `<"john doe"@example.com>` is a
+ * legal msg-id under `obs-id-left` and every mainstream MTA emits and accepts
+ * it. The domain part (`id-right`) stays a plain dot-atom token: this package
+ * mints only dot-atom domains, and widening it further is not needed to
+ * accept ids this package did not author.
+ *
+ * This validates the header SHAPE only, never who may claim it: a frame
+ * arriving on the persist path (`persist.ts`) is never re-validated against
+ * this regex — its `In-Reply-To`/`References` are cached (or dropped) exactly
+ * as decoded, because an external MTA's headers are not this package's frame
+ * to reject.
+ */
+const MSG_ID = /^<(?:[^<>\s"]+|"(?:[^"\\]|\\.)*")@[^<>\s]+>$/;
+
+/**
+ * Refuse a threading header value that is not a bracketed msg-id.
+ *
+ * `RangeError` for the same reason the frame-byte cap throws it: a caller that
+ * hands this builder `uuid@host` (or `<<uuid@host>>`) has a bug, and a frame is
+ * frozen at rest — an unthreadable `References:` written today is unthreadable
+ * forever. Rejecting at the builder is the last point where the caller can
+ * still be blamed precisely.
+ */
+export function assertMsgId(value: string, field: string): void {
+  if (!MSG_ID.test(value)) {
+    throw new RangeError(
+      `mailbox frame ${field} must be a bracketed msg-id (<local@domain>), got ${JSON.stringify(value)}`,
+    );
+  }
+}
+
+// RFC 2822 §2.1.1 caps a line at 78 characters; a References chain on a deep
+// thread passes that within a handful of ids. Folded before each id that would
+// overflow, with the single leading space RFC 2822 §2.2.3 requires — an
+// unfolding parser joins the pieces back into one space-separated value.
+const HEADER_LINE_MAX = 78;
+
+function foldReferences(references: readonly string[]): string {
+  const lines: string[] = ["References:"];
+  for (const reference of references) {
+    const current = lines[lines.length - 1]!;
+    if (current.length + 1 + reference.length > HEADER_LINE_MAX) {
+      lines.push(` ${reference}`);
+      continue;
+    }
+    lines[lines.length - 1] = `${current} ${reference}`;
+  }
+  return lines.join("\r\n");
+}
+
+/** Split an unfolded `References:` value into its msg-ids, oldest first. */
+export function parseMsgIdList(value: string | undefined): string[] {
+  if (value === undefined) return [];
+  return value.match(/<[^<>]+>/g) ?? [];
+}
+
 export type MailFrameArgs = {
   from: string;
   to: string;
@@ -50,6 +114,14 @@ export type MailFrameArgs = {
   messageId: string;
   /** Also a complete msg-id, brackets included. */
   inReplyTo?: string;
+  /**
+   * The thread's ancestry, OLDEST FIRST — the order RFC 2822 §3.6.4 defines
+   * and every threading client walks. Each entry is a complete msg-id,
+   * brackets included. `In-Reply-To` stays a single msg-id (the immediate
+   * parent); this is the whole chain, and the two are independent — a caller
+   * supplying one is not obliged to supply the other.
+   */
+  references?: string[];
 };
 
 /**
@@ -61,15 +133,27 @@ export type MailFrameArgs = {
  */
 export function buildMailFrame(args: MailFrameArgs): Uint8Array {
   const from = headerValue(args.from);
+  const messageId = headerValue(args.messageId);
+  assertMsgId(messageId, "messageId");
   const headers = [
     `From: ${from}`,
     `To: ${headerValue(args.to)}`,
     `Subject: ${headerValue(args.subject)}`,
     `Date: ${formatRFC2822Date(new Date())}`,
-    `Message-ID: ${headerValue(args.messageId)}`,
+    `Message-ID: ${messageId}`,
   ];
   if (args.inReplyTo !== undefined) {
-    headers.push(`In-Reply-To: ${headerValue(args.inReplyTo)}`);
+    const inReplyTo = headerValue(args.inReplyTo);
+    assertMsgId(inReplyTo, "inReplyTo");
+    headers.push(`In-Reply-To: ${inReplyTo}`);
+  }
+  if (args.references !== undefined && args.references.length > 0) {
+    const references = args.references.map((reference) => {
+      const value = headerValue(reference);
+      assertMsgId(value, "references entry");
+      return value;
+    });
+    headers.push(foldReferences(references));
   }
   const body = args.body.replace(/\r?\n/g, "\r\n");
   return new TextEncoder().encode(`${headers.join("\r\n")}\r\n\r\n${body}\r\n`);
@@ -78,6 +162,15 @@ export function buildMailFrame(args: MailFrameArgs): Uint8Array {
 export type DecodedFrame = {
   headers: Map<string, string>;
   body: string;
+  /**
+   * The threading headers, parsed once here rather than re-derived by every
+   * reader. `messageId` and `inReplyTo` are null when the frame carries no such
+   * header; `references` is `[]`, oldest first, for the same case — a chain of
+   * no ancestors is an empty chain, not an absent one.
+   */
+  messageId: string | null;
+  inReplyTo: string | null;
+  references: string[];
 };
 
 function normalizeMailText(bytes: Uint8Array): string {
@@ -144,5 +237,8 @@ export function decodeMailFrame(raw: Uint8Array): DecodedFrame | null {
   return {
     headers: parsed.headers,
     body: extractFrameBody(raw, parsed.headers, parsed.bodyOffset),
+    messageId: parsed.headers.get("message-id") ?? null,
+    inReplyTo: parsed.headers.get("in-reply-to") ?? null,
+    references: parseMsgIdList(parsed.headers.get("references")),
   };
 }