feat: index public-realm deploys and system reindexes without realm events

packages/boxel-cli/plugin/skills/file-ops/SKILL.md

@@ -66,6 +66,7 @@ Write a file to a realm (reads content from STDIN or --file)
 - `--realm <realm-url>` — The realm URL to write to
 - `--file <filepath>` — Read content from a local file instead of STDIN
 - `--realm-secret-seed` — Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED)
+- `--no-realm-events` — Index the write without announcing it to open tabs, which pick it up on their next load (for deploys)
 - `--json` — Output raw JSON response
 
 ### `boxel file list`
@@ -125,5 +126,6 @@ Force realm re-indexing of one or more files by making a semantically-neutral ed
 ## Pitfalls
 
 - `boxel file write` reads content from stdin by default. Pipe in or use `--file` — don't expect interactive prompting.
+- `boxel file write --no-realm-events` follows the same rule as `realm push --no-realm-events`: use it for a deploy into a shared realm, never for a write someone with the realm open needs to see. See [Pushing without realm events](../realm-sync/SKILL.md#pushing-without-realm-events---no-realm-events) in `/boxel-cli:realm-sync`.
 - File paths are realm-relative (e.g., `BlogPost/my-post.json`), not absolute URLs. The realm URL goes in `--realm`.
 - `delete` is unrecoverable from the CLI. Confirm intent before running, especially against production realms.

packages/boxel-cli/src/commands/file/write.ts

@@ -22,13 +22,20 @@ export interface WriteCommandOptions {
   realmSecretSeed?: string;
   /** @internal Test hook: supply an already-constructed authenticator. */
   authenticator?: RealmAuthenticator;
+  /**
+   * When false, the write carries `X-Boxel-Realm-Events: none`, so the realm
+   * indexes it without announcing it to its subscribers. For deploys, whose
+   * changes open tabs pick up on their next load. On by default.
+   */
+  realmEvents?: boolean;
 }
 
 interface WriteCliOptions {
   realm: string;
   file?: string;
   json?: boolean;
   realmSecretSeed?: boolean;
+  realmEvents?: boolean;
 }
 
 /**
@@ -99,12 +106,17 @@ export async function write(
   try {
     let response = await authenticator.authedRealmFetch(url, {
       method: 'POST',
-      headers: isBinary
-        ? { 'Content-Type': SupportedMimeType.OctetStream }
-        : {
-            Accept: SupportedMimeType.CardSource,
-            'Content-Type': SupportedMimeType.CardSource,
-          },
+      headers: {
+        ...(isBinary
+          ? { 'Content-Type': SupportedMimeType.OctetStream }
+          : {
+              Accept: SupportedMimeType.CardSource,
+              'Content-Type': SupportedMimeType.CardSource,
+            }),
+        ...(options?.realmEvents === false
+          ? { 'X-Boxel-Realm-Events': 'none' }
+          : {}),
+      },
       // Both branches of `body: string | Uint8Array` are valid BodyInit
       // values, but TS narrows them as a union that doesn't unify against
       // the fetch signature without a hint.
@@ -158,6 +170,10 @@ export function registerWriteCommand(parent: Command): void {
       '--realm-secret-seed',
       'Administrative auth: prompt for a realm secret seed and mint a JWT locally instead of using a Matrix profile (env: BOXEL_REALM_SECRET_SEED)',
     )
+    .option(
+      '--no-realm-events',
+      'Index the write without announcing it to open tabs, which pick it up on their next load (for deploys)',
+    )
     .option('--json', 'Output raw JSON response')
     .action(async (filePath: string, opts: WriteCliOptions) => {
       // Resolve the seed before consuming stdin: when content arrives on stdin
@@ -221,6 +237,7 @@ export function registerWriteCommand(parent: Command): void {
       try {
         result = await write(opts.realm, filePath, content, {
           realmSecretSeed,
+          realmEvents: opts.realmEvents,
         });
       } catch (err) {
         stderr(

packages/boxel-cli/tests/integration/file-write.test.ts

@@ -1,5 +1,5 @@
 import '../helpers/setup-realm-server.ts';
-import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
@@ -12,6 +12,7 @@ import {
   createTestRealmViaCli,
 } from '../helpers/integration.ts';
 import { runBoxel } from '../helpers/run-boxel.ts';
+import { write } from '../../src/commands/file/write.ts';
 import {
   TINY_PNG_BYTES,
   TINY_PDF_BYTES,
@@ -218,4 +219,58 @@ describe('file write (integration)', () => {
       fs.rmSync(emptyHome, { recursive: true, force: true });
     }
   });
+  it('marks a write sent with realm events off, and leaves a plain write unmarked', async () => {
+    // In-process so the request the write sends can be inspected.
+    let profileManager = reloadProfile(home);
+    let realFetch = profileManager.authedRealmFetch.bind(profileManager);
+    let markers: (string | null)[] = [];
+    let fetchSpy = vi
+      .spyOn(profileManager, 'authedRealmFetch')
+      .mockImplementation(async (input, init) => {
+        if (init?.method === 'POST') {
+          markers.push(new Headers(init?.headers).get('X-Boxel-Realm-Events'));
+        }
+        return realFetch(input, init);
+      });
+
+    try {
+      let plain = await write(realmUrl, 'plain.gts', 'export const p = 1;', {
+        profileManager,
+      });
+      expect(plain.ok, plain.error).toBe(true);
+      let silent = await write(realmUrl, 'silent.gts', 'export const s = 1;', {
+        profileManager,
+        realmEvents: false,
+      });
+      expect(silent.ok, silent.error).toBe(true);
+      let silentBytes = await write(realmUrl, 'silent.png', TINY_PNG_BYTES, {
+        profileManager,
+        realmEvents: false,
+      });
+      expect(silentBytes.ok, silentBytes.error).toBe(true);
+    } finally {
+      fetchSpy.mockRestore();
+    }
+
+    expect(markers).toEqual([null, 'none', 'none']);
+    expect(await (await readBack('silent.gts')).text()).toContain('s = 1');
+  });
+
+  // The header itself is checked in-process above; this checks that the CLI
+  // accepts the flag and still writes.
+  it('accepts --no-realm-events and writes the file', async () => {
+    let res = await runBoxel(
+      [
+        'file',
+        'write',
+        'flagged.gts',
+        '--realm',
+        realmUrl,
+        '--no-realm-events',
+      ],
+      { home, input: 'export const flagged = 1;' },
+    );
+    expect(res.ok, res.stderr).toBe(true);
+    expect(await (await readBack('flagged.gts')).text()).toContain('flagged');
+  });
 });

packages/host/app/services/card-service.ts

@@ -6,6 +6,7 @@ import { v4 as uuidv4 } from 'uuid';
 import {
   formattedError,
   isJsonContentType,
+  REALM_EVENTS_HEADER,
   SupportedMimeType,
   type CardDocument,
   type SingleCardDocument,
@@ -438,7 +439,14 @@ export default class CardService extends Service {
   async executeAtomicOperations(
     operations: AtomicOperation[],
     realm: RealmIdentifier | URL,
-    opts?: { clientRequestId?: string },
+    opts?: {
+      clientRequestId?: string;
+      // False for a write the system makes on its own, such as a scheduled
+      // sync of a public realm's content. The realm indexes it as usual but
+      // announces nothing to its subscribers, so no open tab resets its loader
+      // for a change its user did not ask for (see `REALM_EVENTS_HEADER`).
+      realmEvents?: boolean;
+    },
   ) {
     let realmURL = this.toURL(realm);
     for (let operation of operations) {
@@ -463,6 +471,9 @@ export default class CardService extends Service {
         ...(opts?.clientRequestId
           ? { 'X-Boxel-Client-Request-Id': opts.clientRequestId }
           : {}),
+        ...(opts?.realmEvents === false
+          ? { [REALM_EVENTS_HEADER]: 'none' }
+          : {}),
       },
       body: JSON.stringify(doc),
     });

packages/host/app/services/store.ts

@@ -2565,10 +2565,12 @@ export default class StoreService extends Service implements StoreInterface {
     if (event.indexType === 'full') {
       // A full reindex carries no per-file invalidation list, so there is
       // nothing to reload by name. A realm that reindexes on request does
-      // broadcast the URLs it visited as an incremental event first, but one
-      // reindexing at startup announces itself with this event alone — so this
-      // can be the only word a card being held as awaiting-index ever gets
-      // that the row it is waiting for now exists.
+      // broadcast the URLs it visited as an incremental event first, but a
+      // realm building a brand-new index at startup announces itself with
+      // this event alone — so this can be the only word a card being held as
+      // awaiting-index ever gets that the row it is waiting for now exists.
+      // A realm reindexing an existing index at startup, as on a deploy,
+      // announces nothing.
       let reloadsTriggered = this.reloadAwaitingIndexInstances(event.realmURL);
       // Report the pass as a thin realm-event so the dashboard still sees it.
       telemetry?.recordEvent({

packages/host/app/tools/sync-openrouter-models.ts

@@ -271,13 +271,16 @@ export default class SyncOpenRouterModelsTool extends HostBaseTool<
     });
   }
 
+  // The sync is a scheduled refresh of a public realm that no reader asked
+  // for, so its writes announce nothing to the realm's subscribers.
   private async executeBatchWithRetry(
     batch: AtomicOperation[],
     realmURL: URL,
   ): Promise<{ ok: boolean; error?: string }> {
     let result = await this.cardService.executeAtomicOperations(
       batch,
       realmURL,
+      { realmEvents: false },
     );
 
     if (!result.errors) {
@@ -325,6 +328,7 @@ export default class SyncOpenRouterModelsTool extends HostBaseTool<
     let retryResult = await this.cardService.executeAtomicOperations(
       retryBatch,
       realmURL,
+      { realmEvents: false },
     );
 
     if (retryResult.errors) {

packages/realm-server/handlers/handle-full-reindex.ts

@@ -8,6 +8,9 @@ import { setContextResponse } from '../middleware/index.ts';
 import { getFullReindexRealmUrls } from '../lib/full-reindex-realm-urls.ts';
 import type { CreateRoutesArgs } from '../routes.ts';
 
+// An operator's reindex of every realm. None of the realms' users asked for
+// it, so no realm's pass announces anything to its subscribers (see
+// `fromScratchAnnounces`).
 export default function handleFullReindex({
   dbAdapter,
   queue,
@@ -25,6 +28,7 @@ export default function handleFullReindex({
       priority: systemInitiatedPriority,
       args: {
         realmUrls,
+        realmEvents: false,
       },
     });
     await setContextResponse(

packages/realm-server/handlers/handle-post-deployment.ts

@@ -45,13 +45,17 @@ export default function handlePostDeployment({
     ) {
       let realmUrls = await getFullReindexRealmUrls(dbAdapter);
 
+      // A deploy's reindex: no realm's users asked for it, so no realm's pass
+      // announces anything to its subscribers (see
+      // `fromScratchAnnounces`).
       await queue.publish<void>({
         jobType: `full-reindex`,
         concurrencyGroup: `full-reindex-group`,
         timeout: FULL_REINDEX_JOB_TIMEOUT_SEC,
         priority: systemInitiatedPriority,
         args: {
           realmUrls,
+          realmEvents: false,
         },
       });
 

packages/realm-server/handlers/handle-reindex.ts

@@ -101,6 +101,9 @@ export default function handleReindex({
   };
 }
 
+// An operator's reindex, which the realm's users did not ask for, so it
+// announces nothing to the realm's subscribers (see
+// `fromScratchAnnounces`).
 export async function reindex({
   realm,
   queue,
@@ -123,6 +126,7 @@ export async function reindex({
     priority,
     {
       clearLastModified: true,
+      realmEvents: false,
     },
   );
 }

packages/realm-server/tests/atomic-endpoints-test.ts

@@ -1448,6 +1448,86 @@ module(basename(import.meta.filename), function () {
           );
         });
 
+        test('an ignore-file write sent with X-Boxel-Realm-Events: none reindexes the realm without any realm event', async function (assert) {
+          let auth = `Bearer ${createJWT(testRealm, 'user', ['read', 'write'])}`;
+          let writeIgnoreFile = async (
+            content: string,
+            headers: Record<string, string>,
+          ) => {
+            let pending = request
+              .post('/.boxelignore')
+              .set('Accept', SupportedMimeType.CardSource)
+              .set('Content-Type', 'text/plain;charset=UTF-8')
+              .set('Authorization', auth);
+            for (let [name, value] of Object.entries(headers)) {
+              pending = pending.set(name, value);
+            }
+            let response = await pending.send(content);
+            assert.strictEqual(response.status, 204);
+          };
+
+          let [{ max: jobsBefore }] = (await dbAdapter.execute(
+            'SELECT coalesce(max(id), 0) AS max FROM jobs',
+          )) as { max: number }[];
+          let events = recordRealmEvents();
+          await writeIgnoreFile('quiet-ignored.json\n', {
+            'X-Boxel-Realm-Events': 'none',
+          });
+          // The write's own pass, then the from-scratch pass it starts.
+          await testRealm.indexing();
+          await testRealm.indexing();
+          // The reindex announces right after its pass settles.
+          await new Promise((resolve) => setTimeout(resolve, 250));
+          assert.deepEqual(
+            eventKinds(events),
+            [],
+            'neither the write nor the reindex it started sent a realm event',
+          );
+          // What the worker would announce is carried by the jobs the pass
+          // spawned: its render job and any follow-up pass.
+          let spawned = (await dbAdapter.execute(
+            `SELECT job_type, args FROM jobs
+             WHERE id > $1
+               AND job_type IN ('from-scratch-index', 'prerender_html')
+             ORDER BY id`,
+            { bind: [jobsBefore] },
+          )) as {
+            job_type: string;
+            args: { realmEvents?: boolean; announce?: boolean };
+          }[];
+          let reindex = spawned.find(
+            ({ job_type }) => job_type === 'from-scratch-index',
+          );
+          assert.ok(reindex, 'the ignore-file write started a reindex');
+          assert.false(
+            reindex?.args.realmEvents,
+            'the reindex announces nothing',
+          );
+          assert.deepEqual(
+            spawned
+              .filter(({ job_type }) => job_type === 'prerender_html')
+              .map(({ args }) => args.announce),
+            spawned
+              .filter(({ job_type }) => job_type === 'prerender_html')
+              .map(() => false),
+            'no render job the passes spawned announces',
+          );
+
+          await writeIgnoreFile('loud-ignored.json\n', {});
+          await waitUntil(
+            async () => eventKinds(events).includes('index:full'),
+            {
+              timeout: 30000,
+              timeoutMessage: () =>
+                `no full index event; saw ${JSON.stringify(eventKinds(events))}`,
+            },
+          );
+          assert.ok(
+            eventKinds(events).includes('index:full'),
+            'an ignore-file write without the header announces the reindex',
+          );
+        });
+
         test('removing several files at once sends one initiation', async function (assert) {
           await postAtomic(peopleBatch(['gone-a', 'gone-b', 'gone-c']));
           await testRealm.incrementalIndexing();

packages/realm-server/tests/full-reindex-test.ts

@@ -188,6 +188,25 @@ module(basename(import.meta.filename), function (hooks) {
     );
   });
 
+  test('a sweep that announces nothing enqueues passes that announce nothing', async function (assert) {
+    const sourceRealmURL = 'http://example.com/source/';
+    await insertPermissions(dbAdapter, new URL(sourceRealmURL), {
+      '@owner:localhost': ['read', 'realm-owner'],
+    });
+
+    let reindex = buildFullReindexTask();
+    await reindex({ realmUrls: [sourceRealmURL], realmEvents: false });
+
+    let jobs = (await dbAdapter.execute('select * from jobs')) as {
+      args: { realmURL: string; realmEvents?: boolean };
+    }[];
+    assert.strictEqual(jobs.length, 1, 'a from-scratch job was enqueued');
+    assert.false(
+      jobs[0].args.realmEvents,
+      "the realm's pass announces nothing to its subscribers",
+    );
+  });
+
   test('enqueues the base realm above the system tier, every other realm at it', async function (assert) {
     const sourceRealmURL = 'http://example.com/source/';